darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-24 01:11:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1863 from alphagov/use-ft-billing-for-usage

Browse Source 
Use ft billing for usage
This commit is contained in:
Rebecca Law
2018-05-08 17:00:56 +01:00
committed by GitHub
parent 54d0e64582 844f5b2e5d
commit d52e65d89a
7 changed files with 352 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/billing/billing_schemas.py
View File

@@ -1,3 +1,5 @@
from datetime import datetime
create_or_update_free_sms_fragment_limit_schema = {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "POST annual billing schema",
@@ -8,3 +10,17 @@ create_or_update_free_sms_fragment_limit_schema = {
},
"required": ["free_sms_fragment_limit"]
}
def serialize_ft_billing_remove_emails(data):
results = []
billed_notifications = [x for x in data if x.notification_type != 'email']
for notifications in billed_notifications:
json_result = {
"month": (datetime.strftime(notifications.month, "%B")),
"notification_type": notifications.notification_type,
"billing_units": int(notifications.billable_units),
"rate": float(notifications.rate),
}
results.append(json_result)
return results

45
app/billing/rest.py
View File

@@ -1,24 +1,30 @@
from datetime import datetime
import json
from datetime import datetime
from flask import Blueprint, jsonify, request
from app.billing.billing_schemas import (
create_or_update_free_sms_fragment_limit_schema,
serialize_ft_billing_remove_emails
)
from app.dao.annual_billing_dao import (
dao_get_free_sms_fragment_limit_for_year,
dao_get_all_free_sms_fragment_limit,
dao_create_or_update_annual_billing_for_year,
dao_update_annual_billing_for_future_years
)
from app.dao.date_util import get_current_financial_year_start_year
from app.dao.date_util import get_months_for_financial_year
from app.dao.fact_billing_dao import fetch_monthly_billing_for_year
from app.dao.monthly_billing_dao import (
get_billing_data_for_financial_year,
get_monthly_billing_by_notification_type
)
from app.dao.date_util import get_months_for_financial_year
from app.errors import InvalidRequest
from app.errors import register_errors
from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
from app.utils import convert_utc_to_bst
from app.dao.annual_billing_dao import (dao_get_free_sms_fragment_limit_for_year,
dao_get_all_free_sms_fragment_limit,
dao_create_or_update_annual_billing_for_year,
dao_update_annual_billing_for_future_years)
from app.billing.billing_schemas import create_or_update_free_sms_fragment_limit_schema
from app.errors import InvalidRequest
from app.schema_validation import validate
from app.dao.date_util import get_current_financial_year_start_year
from app.utils import convert_utc_to_bst
billing_blueprint = Blueprint(
'billing',
@@ -30,19 +36,30 @@ billing_blueprint = Blueprint(
register_errors(billing_blueprint)
@billing_blueprint.route('/ft-monthly-usage')
def get_yearly_usage_by_monthly_from_ft_billing(service_id):
try:
year = int(request.args.get('year'))
except TypeError:
return jsonify(result='error', message='No valid year provided'), 400
results = fetch_monthly_billing_for_year(service_id=service_id, year=year)
data = serialize_ft_billing_remove_emails(results)
return jsonify(data)
@billing_blueprint.route('/monthly-usage')
def get_yearly_usage_by_month(service_id):
try:
year = int(request.args.get('year'))
results = []
for month in get_months_for_financial_year(year):
billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
if billing_for_month:
results.append(_transform_billing_for_month_sms(billing_for_month))
letter_billing_for_month = get_monthly_billing_by_notification_type(service_id, month, LETTER_TYPE)
if letter_billing_for_month:
results.extend(_transform_billing_for_month_letters(letter_billing_for_month))
return json.dumps(results)
billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
if billing_for_month:
results.append(_transform_billing_for_month_sms(billing_for_month))
return jsonify(results)
except TypeError:
return jsonify(result='error', message='No valid year provided'), 400

43
app/commands.py
View File

@@ -7,16 +7,18 @@ from decimal import Decimal
import click
import flask
from click_datetime import Datetime as click_dt
from flask import current_app
from flask import current_app, json
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy import func
from notifications_utils.statsd_decorators import statsd
from app import db, DATETIME_FORMAT, encryption, redis_store
from app.billing.rest import get_yearly_usage_by_month, get_yearly_usage_by_monthly_from_ft_billing
from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.celery.letters_pdf_tasks import create_letters_pdf
from app.config import QueueNames
from app.dao.date_util import get_financial_year
from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
from app.dao.monthly_billing_dao import (
create_or_update_monthly_billing,
@@ -562,3 +564,42 @@ def rebuild_ft_billing_for_month_and_service(service_id, day):
transit_data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
for data in transit_data:
update_fact_billing(data, day)
@notify_command(name='compare-ft-billing-to-monthly-billing')
@click.option('-y', '--year', required=True)
@click.option('-s', '--service_id', required=False, type=click.UUID)
def compare_ft_billing_to_monthly_billing(year, service_id=None):
"""
This command checks the results of monthly_billing to ft_billing for the given year.
If service id is not included all services are compared for the given year.
"""
def compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response):
# Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows.
mo_json = json.loads(monthly_billing_response.get_data(as_text=True))
rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0]
try:
assert rm_zero_rows == json.loads(ft_billing_response.get_data(as_text=True))
except AssertionError:
print("Comparison failed for service: {} and year: {}".format(service_id, year))
if not service_id:
start_date, end_date = get_financial_year(year=int(year))
services = get_service_ids_that_need_billing_populated(start_date, end_date)
for service_id in services:
with current_app.test_request_context(
path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
monthly_billing_response = get_yearly_usage_by_month(service_id)
with current_app.test_request_context(
path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
else:
with current_app.test_request_context(
path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
monthly_billing_response = get_yearly_usage_by_month(service_id)
with current_app.test_request_context(
path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)

26
app/dao/fact_billing_dao.py
View File

@@ -1,7 +1,7 @@
from datetime import datetime, timedelta, time
from flask import current_app
from sqlalchemy import func, case, desc, extract
from sqlalchemy import func, case, desc, Date
from app import db
from app.dao.date_util import get_financial_year
@@ -20,12 +20,12 @@ from app.models import (
from app.utils import convert_utc_to_bst, convert_bst_to_utc
def fetch_montly_billing_for_year(service_id, year):
def fetch_monthly_billing_for_year(service_id, year):
year_start_date, year_end_date = get_financial_year(year)
utcnow = datetime.utcnow()
today = convert_utc_to_bst(utcnow).date()
today = convert_utc_to_bst(utcnow)
# if year end date is less than today, we are calculating for data in the past and have no need for deltas.
if year_end_date.date() >= today:
if year_end_date >= today:
yesterday = today - timedelta(days=1)
for day in [yesterday, today]:
data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
@@ -33,23 +33,25 @@ def fetch_montly_billing_for_year(service_id, year):
update_fact_billing(data=d, process_day=day)
yearly_data = db.session.query(
extract('month', FactBilling.bst_date).label("Month"),
func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
func.sum(FactBilling.notifications_sent).label("notifications_sent"),
func.sum(FactBilling.billable_units).label("billable_units"),
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
FactBilling.service_id,
FactBilling.rate,
FactBilling.rate_multiplier,
FactBilling.international
FactBilling.notification_type
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start_date,
FactBilling.bst_date <= year_end_date
).group_by(
'Month',
'month',
FactBilling.service_id,
FactBilling.rate,
FactBilling.rate_multiplier,
FactBilling.international
FactBilling.notification_type
).order_by(
FactBilling.service_id,
'month',
FactBilling.notification_type
).all()
return yearly_data
@@ -119,7 +121,7 @@ def update_fact_billing(data, process_day):
inserted_records = 0
updated_records = 0
non_letter_rates, letter_rates = get_rates_for_billing()
print("process_day: {} {}".format(type(process_day), process_day))
update_count = FactBilling.query.filter(
FactBilling.bst_date == datetime.date(process_day),
FactBilling.template_id == data.template_id,

3
app/dao/monthly_billing_dao.py
View File

@@ -100,7 +100,8 @@ def get_yearly_billing_data_for_date_range(
MonthlyBilling.end_date <= end_date,
MonthlyBilling.notification_type.in_(notification_types)
).order_by(
MonthlyBilling.notification_type
MonthlyBilling.start_date,
MonthlyBilling.notification_type,
).all()
return results

170
tests/app/billing/test_billing.py
View File

@@ -1,3 +1,4 @@
from calendar import monthrange
from datetime import datetime, timedelta
import json
@@ -8,8 +9,8 @@ from app.dao.monthly_billing_dao import (
create_or_update_monthly_billing,
get_monthly_billing_by_notification_type,
)
from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
from app.dao.date_util import get_current_financial_year_start_year
from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, FactBilling
from app.dao.date_util import get_current_financial_year_start_year, get_month_start_and_end_date_in_utc
from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year
from tests.app.db import (
create_notification,
@@ -17,7 +18,9 @@ from tests.app.db import (
create_monthly_billing_entry,
create_annual_billing,
create_letter_rate,
create_template
create_template,
create_service,
create_ft_billing
)
from app.billing.rest import update_free_sms_fragment_limit_data
@@ -141,30 +144,29 @@ def test_get_yearly_usage_by_month_returns_correctly(client, sample_template):
resp_json = json.loads(response.get_data(as_text=True))
_assert_dict_equals(resp_json[0], {
'billing_units': 0,
'month': 'May',
'notification_type': LETTER_TYPE,
'rate': 0
})
_assert_dict_equals(resp_json[1], {
'billing_units': 2,
'month': 'May',
'notification_type': SMS_TYPE,
'rate': 0.12
})
_assert_dict_equals(resp_json[1], {
_assert_dict_equals(resp_json[2], {
'billing_units': 0,
'month': 'May',
'month': 'June',
'notification_type': LETTER_TYPE,
'rate': 0
})
_assert_dict_equals(resp_json[2], {
_assert_dict_equals(resp_json[3], {
'billing_units': 6,
'month': 'June',
'notification_type': SMS_TYPE,
'rate': 0.12
})
_assert_dict_equals(resp_json[3], {
'billing_units': 0,
'month': 'June',
'notification_type': LETTER_TYPE,
'rate': 0
})
def test_transform_billing_for_month_returns_empty_if_no_monthly_totals(sample_service):
@@ -409,3 +411,145 @@ def test_update_free_sms_fragment_limit_data(client, sample_service):
annual_billing = dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year)
assert annual_billing.free_sms_fragment_limit == 9999
def test_get_yearly_usage_by_monthly_from_ft_billing_populates_deltas(client, notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type="sms")
create_rate(start_date=datetime.utcnow() - timedelta(days=1), value=0.158, notification_type='sms')
create_notification(template=sms_template, status='delivered')
assert FactBilling.query.count() == 0
response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id),
headers=[('Content-Type', 'application/json'), create_authorization_header()])
assert response.status_code == 200
assert len(json.loads(response.get_data(as_text=True))) == 1
fact_billing = FactBilling.query.all()
assert len(fact_billing) == 1
assert fact_billing[0].notification_type == 'sms'
def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type="sms")
email_template = create_template(service=service, template_type="email")
letter_template = create_template(service=service, template_type="letter")
for month in range(1, 13):
mon = str(month).zfill(2)
for day in range(1, monthrange(2016, month)[1] + 1):
d = str(day).zfill(2)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=sms_template,
notification_type='sms',
billable_unit=1,
rate=0.162)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=email_template,
notification_type='email',
rate=0)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=letter_template,
notification_type='letter',
billable_unit=1,
rate=0.33)
response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
headers=[('Content-Type', 'application/json'), create_authorization_header()])
json_resp = json.loads(response.get_data(as_text=True))
ft_letters = [x for x in json_resp if x['notification_type'] == 'letter']
ft_sms = [x for x in json_resp if x['notification_type'] == 'sms']
ft_email = [x for x in json_resp if x['notification_type'] == 'email']
keys = [x.keys() for x in ft_sms][0]
expected_sms_april = {"month": "April",
"notification_type": "sms",
"billing_units": 30,
"rate": 0.162
}
expected_letter_april = {"month": "April",
"notification_type": "letter",
"billing_units": 30,
"rate": 0.33
}
for k in keys:
assert ft_sms[0][k] == expected_sms_april[k]
assert ft_letters[0][k] == expected_letter_april[k]
assert len(ft_email) == 0
def test_compare_ft_billing_to_monthly_billing(client, notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type="sms")
email_template = create_template(service=service, template_type="email")
letter_template = create_template(service=service, template_type="letter")
for month in range(1, 13):
mon = str(month).zfill(2)
for day in range(1, monthrange(2016, month)[1] + 1):
d = str(day).zfill(2)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=sms_template,
notification_type='sms',
rate=0.0162)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=sms_template,
notification_type='sms',
rate_multiplier=2,
rate=0.0162)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=email_template,
notification_type='email',
rate=0)
create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
service=service,
template=letter_template,
notification_type='letter',
rate=0.33)
start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1))
create_monthly_billing_entry(service=service, start_date=start_date,
end_date=end_date,
notification_type='sms',
monthly_totals=[
{"rate": 0.0162, "international": False,
"rate_multiplier": 1, "billing_units": int(d),
"total_cost": 0.0162 * int(d)},
{"rate": 0.0162, "international": False,
"rate_multiplier": 2, "billing_units": int(d),
"total_cost": 0.0162 * int(d)}]
)
create_monthly_billing_entry(service=service, start_date=start_date,
end_date=end_date,
notification_type='email',
monthly_totals=[
{"rate": 0, "international": False,
"rate_multiplier": 1, "billing_units": int(d),
"total_cost": 0}]
)
create_monthly_billing_entry(service=service, start_date=start_date,
end_date=end_date,
notification_type='letter',
monthly_totals=[
{"rate": 0.33, "international": False,
"rate_multiplier": 1, "billing_units": int(d),
"total_cost": 0.33 * int(d)}]
)
monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id),
headers=[create_authorization_header()])
ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
headers=[('Content-Type', 'application/json'), create_authorization_header()])
monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
assert monthly_billing_json_resp == ft_billing_json_resp

110
tests/app/dao/test_ft_billing_dao.py
View File

@@ -1,3 +1,4 @@
from calendar import monthrange
from decimal import Decimal
from datetime import datetime, timedelta
@@ -5,7 +6,7 @@ from freezegun import freeze_time
from app import db
from app.dao.fact_billing_dao import (
fetch_montly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing,
fetch_monthly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing,
get_rate
)
from app.models import FactBilling
@@ -132,6 +133,25 @@ def test_fetch_billing_data_for_day_is_grouped_by_international(notify_db_sessio
assert results[1].notifications_sent == 1
def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type='sms')
email_template = create_template(service=service, template_type='email')
letter_template = create_template(service=service, template_type='letter')
create_notification(template=sms_template, status='delivered')
create_notification(template=sms_template, status='delivered')
create_notification(template=sms_template, status='delivered')
create_notification(template=email_template, status='delivered')
create_notification(template=email_template, status='delivered')
create_notification(template=letter_template, status='delivered')
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
assert len(results) == 3
notification_types = [x[2] for x in results if x[2] in ['email', 'sms', 'letter']]
assert len(notification_types) == 3
def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session):
today = convert_utc_to_bst(datetime.utcnow())
results = fetch_billing_data_for_day(today)
@@ -181,7 +201,7 @@ def test_get_rate(notify_db_session):
assert letter_rate == Decimal('4.4')
def test_fetch_annual_billing_for_year(notify_db_session):
def test_fetch_monthly_billing_for_year(notify_db_session):
service = create_service()
template = create_template(service=service, template_type="sms")
for i in range(1, 31):
@@ -189,6 +209,7 @@ def test_fetch_annual_billing_for_year(notify_db_session):
service=service,
template=template,
notification_type='sms',
rate_multiplier=2,
rate=0.162)
for i in range(1, 32):
create_ft_billing(bst_date='2018-07-{}'.format(i),
@@ -197,28 +218,26 @@ def test_fetch_annual_billing_for_year(notify_db_session):
notification_type='sms',
rate=0.158)
results = fetch_montly_billing_for_year(service_id=service.id, year=2018)
results = fetch_monthly_billing_for_year(service_id=service.id, year=2018)
assert len(results) == 2
assert results[0][0] == 6.0
assert results[0][1] == 30
assert results[0][2] == Decimal('30')
assert results[0][3] == service.id
assert results[0][4] == Decimal('0.162')
assert results[0][5] == Decimal('1')
assert results[0][6] is False
assert str(results[0].month) == "2018-06-01"
assert results[0].notifications_sent == 30
assert results[0].billable_units == Decimal('60')
assert results[0].service_id == service.id
assert results[0].rate == Decimal('0.162')
assert results[0].notification_type == 'sms'
assert results[1][0] == 7.0
assert results[1][1] == 31
assert results[1][2] == Decimal('31')
assert results[1][3] == service.id
assert results[1][4] == Decimal('0.158')
assert results[1][5] == Decimal('1')
assert results[1][6] is False
assert str(results[1].month) == "2018-07-01"
assert results[1].notifications_sent == 31
assert results[1].billable_units == Decimal('31')
assert results[1].service_id == service.id
assert results[1].rate == Decimal('0.158')
assert results[1].notification_type == 'sms'
@freeze_time('2018-08-01 13:30:00')
def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session):
def test_fetch_monthly_billing_for_year_adds_data_for_today(notify_db_session):
service = create_service()
template = create_template(service=service, template_type="email")
for i in range(1, 32):
@@ -230,7 +249,58 @@ def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session):
create_notification(template=template, status='delivered')
assert db.session.query(FactBilling.bst_date).count() == 31
results = fetch_montly_billing_for_year(service_id=service.id,
year=2018)
results = fetch_monthly_billing_for_year(service_id=service.id,
year=2018)
assert db.session.query(FactBilling.bst_date).count() == 32
assert len(results) == 2
def test_fetch_monthly_billing_for_year_return_financial_year(notify_db_session):
service = create_service()
sms_template = create_template(service=service, template_type="sms")
email_template = create_template(service=service, template_type="email")
letter_template = create_template(service=service, template_type="letter")
for year in (2016, 2017):
for month in range(1, 13):
mon = str(month).zfill(2)
for day in range(1, monthrange(year, month)[1] + 1):
d = str(day).zfill(2)
create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
service=service,
template=sms_template,
notification_type='sms',
rate=0.162)
create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
service=service,
template=email_template,
notification_type='email',
rate=0)
create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
service=service,
template=letter_template,
notification_type='letter',
rate=0.33)
results = fetch_monthly_billing_for_year(service.id, 2016)
# returns 3 rows, per month, returns financial year april to end of march
# Orders by Month
assert len(results) == 36
assert str(results[0].month) == "2016-04-01"
assert results[0].notification_type == 'email'
assert results[0].notifications_sent == 30
assert results[0].billable_units == 30
assert results[0].rate == Decimal('0')
assert str(results[1].month) == "2016-04-01"
assert results[1].notification_type == 'letter'
assert results[1].notifications_sent == 30
assert results[1].billable_units == 30
assert results[1].rate == Decimal('0.33')
assert str(results[2].month) == "2016-04-01"
assert results[2].notification_type == 'sms'
assert results[2].notifications_sent == 30
assert results[2].billable_units == 30
assert results[2].rate == Decimal('0.162')
assert str(results[3].month) == "2016-05-01"
assert str(results[35].month) == "2017-03-01"