Merge pull request #2723 from alphagov/organisation-usage

Organisation usage

app/dao/fact_billing_dao.py

@@ -12,6 +12,7 @@ from app.dao.date_util import (
     get_financial_year,
     get_financial_year_for_datetime
 )
+from app.dao.organisation_dao import dao_get_organisation_live_services
 
 from app.models import (
     FactBilling,
@@ -243,11 +244,9 @@ def fetch_monthly_billing_for_year(service_id, year):
     today = convert_utc_to_bst(datetime.utcnow()).date()
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
     if year_end_date >= today:
-        yesterday = today - timedelta(days=1)
+        data = fetch_billing_data_for_day(process_day=today, service_id=service_id, check_permissions=True)
-        for day in [yesterday, today]:
+        for d in data:
-            data = fetch_billing_data_for_day(process_day=day, service_id=service_id, check_permissions=True)
+            update_fact_billing(data=d, process_day=today)
-            for d in data:
-                update_fact_billing(data=d, process_day=day)
 
     email_and_letters = db.session.query(
         func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
@@ -539,3 +538,150 @@ def create_billing_record(data, rate, process_day):
         postage=data.postage,
     )
     return billing_record
+
+
+@statsd(namespace="dao")
+def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date):
+    query = db.session.query(
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
+    ).select_from(
+        Service
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == LETTER_TYPE,
+        Service.organisation_id == organisation_id
+    ).group_by(
+        Service.id,
+        Service.name,
+    ).order_by(
+        Service.name
+    )
+
+    return query.all()
+
+
+@statsd(namespace="dao")
+def fetch_email_usage_for_organisation(organisation_id, start_date, end_date):
+    query = db.session.query(
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        func.sum(FactBilling.notifications_sent).label("emails_sent")
+    ).select_from(
+        Service
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == EMAIL_TYPE,
+        Service.organisation_id == organisation_id
+    ).group_by(
+        Service.id,
+        Service.name,
+    ).order_by(
+        Service.name
+    )
+    return query.all()
+
+
+@statsd(namespace="dao")
+def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery()
+
+    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+    sms_remainder = func.coalesce(
+        free_allowance_remainder.c.sms_remainder,
+        free_allowance_remainder.c.free_sms_fragment_limit
+    )
+    chargeable_sms = func.greatest(sms_billable_units - sms_remainder, 0)
+    sms_cost = chargeable_sms * FactBilling.rate
+
+    query = db.session.query(
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        FactBilling.rate.label('sms_rate'),
+        sms_remainder.label("sms_remainder"),
+        sms_billable_units.label('sms_billable_units'),
+        chargeable_sms.label("chargeable_billable_sms"),
+        sms_cost.label('sms_cost'),
+    ).select_from(
+        Service
+    ).outerjoin(
+        free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == SMS_TYPE,
+        Service.organisation_id == organisation_id
+    ).group_by(
+        Service.id,
+        Service.name,
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        free_allowance_remainder.c.sms_remainder,
+        FactBilling.rate,
+    ).order_by(
+        Service.name
+    )
+
+    return query.all()
+
+
+@statsd(namespace="dao")
+def fetch_usage_year_for_organisation(organisation_id, year):
+    year_start_datetime, year_end_datetime = get_financial_year(year)
+
+    year_start_date = convert_utc_to_bst(year_start_datetime).date()
+    year_end_date = convert_utc_to_bst(year_end_datetime).date()
+
+    today = convert_utc_to_bst(datetime.utcnow()).date()
+    services = dao_get_organisation_live_services(organisation_id)
+    # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
+    if year_end_date >= today:
+        for service in services:
+            data = fetch_billing_data_for_day(process_day=today, service_id=service.id)
+            for d in data:
+                update_fact_billing(data=d, process_day=today)
+    service_with_usage = {}
+    # initialise results
+    for service in services:
+        service_with_usage[str(service.id)] = {
+            'service_id': service.id,
+            'service_name': service.name,
+            'free_sms_limit': 0,
+            'sms_remainder': 0,
+            'sms_billable_units': 0,
+            'chargeable_billable_sms': 0,
+            'sms_cost': 0.0,
+            'letter_cost': 0.0,
+            'emails_sent': 0
+        }
+    sms_usages = fetch_sms_billing_for_organisation(organisation_id, year_start_date, year_end_date)
+    letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start_date, year_end_date)
+    email_usages = fetch_email_usage_for_organisation(organisation_id, year_start_date, year_end_date)
+    for usage in sms_usages:
+        service_with_usage[str(usage.service_id)] = {
+            'service_id': usage.service_id,
+            'service_name': usage.service_name,
+            'free_sms_limit': usage.free_sms_fragment_limit,
+            'sms_remainder': usage.sms_remainder,
+            'sms_billable_units': usage.sms_billable_units,
+            'chargeable_billable_sms': usage.chargeable_billable_sms,
+            'sms_cost': float(usage.sms_cost),
+            'letter_cost': 0.0,
+            'emails_sent': 0
+        }
+    for letter_usage in letter_usages:
+        service_with_usage[str(letter_usage.service_id)]['letter_cost'] = float(letter_usage.letter_cost)
+    for email_usage in email_usages:
+        service_with_usage[str(email_usage.service_id)]['emails_sent'] = email_usage.emails_sent
+
+    return service_with_usage

app/dao/organisation_dao.py

@@ -17,7 +17,7 @@ def dao_get_organisations():
     ).all()
 
 
-def dao_count_organsations_with_live_services():
+def dao_count_organisations_with_live_services():
     return db.session.query(Organisation.id).join(Organisation.services).filter(
         Service.active.is_(True),
         Service.restricted.is_(False),
@@ -31,6 +31,13 @@ def dao_get_organisation_services(organisation_id):
     ).one().services
 
 
+def dao_get_organisation_live_services(organisation_id):
+    return Service.query.filter_by(
+        organisation_id=organisation_id,
+        restricted=False
+    ).all()
+
+
 def dao_get_organisation_by_id(organisation_id):
     return Organisation.query.filter_by(id=organisation_id).one()
 

app/organisation/rest.py

@@ -1,7 +1,9 @@
+
 from flask import abort, Blueprint, jsonify, request, current_app
 from sqlalchemy.exc import IntegrityError
 
 from app.config import QueueNames
+from app.dao.fact_billing_dao import fetch_usage_year_for_organisation
 from app.dao.organisation_dao import (
     dao_create_organisation,
     dao_get_organisations,
@@ -125,6 +127,18 @@ def get_organisation_services(organisation_id):
     return jsonify([s.serialize_for_org_dashboard() for s in sorted_services])
 
 
+@organisation_blueprint.route('/<uuid:organisation_id>/services-with-usage', methods=['GET'])
+def get_organisation_services_usage(organisation_id):
+    try:
+        year = int(request.args.get('year', 'none'))
+    except ValueError:
+        return jsonify(result='error', message='No valid year provided'), 400
+    services = fetch_usage_year_for_organisation(organisation_id, year)
+    list_services = services.values()
+    sorted_services = sorted(list_services, key=lambda s: s['service_name'].lower())
+    return jsonify(services=sorted_services)
+
+
 @organisation_blueprint.route('/<uuid:organisation_id>/users/<uuid:user_id>', methods=['POST'])
 def add_user_to_organisation(organisation_id, user_id):
     new_org_user = dao_add_user_to_organisation(organisation_id, user_id)

app/status/healthcheck.py

@@ -6,7 +6,7 @@ from flask import (
 
 from app import db, version
 from app.dao.services_dao import dao_count_live_services
-from app.dao.organisation_dao import dao_count_organsations_with_live_services
+from app.dao.organisation_dao import dao_count_organisations_with_live_services
 
 status = Blueprint('status', __name__)
 
@@ -28,7 +28,7 @@ def show_status():
 @status.route('/_status/live-service-and-organisation-counts')
 def live_service_and_organisation_counts():
     return jsonify(
-        organisations=dao_count_organsations_with_live_services(),
+        organisations=dao_count_organisations_with_live_services(),
         services=dao_count_live_services(),
     ), 200
 

tests/app/dao/test_ft_billing_dao.py

@@ -18,7 +18,10 @@ from app.dao.fact_billing_dao import (
     get_rates_for_billing,
     fetch_sms_free_allowance_remainder,
     fetch_sms_billing_for_all_services,
-    fetch_letter_costs_for_all_services, fetch_letter_line_items_for_all_services)
+    fetch_letter_costs_for_all_services,
+    fetch_letter_line_items_for_all_services,
+    fetch_usage_year_for_organisation
+)
 from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.models import (
     FactBilling,
@@ -582,8 +585,6 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
 
     results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31))
     assert len(results) == 3
-    # (organisation_name, organisation_id, service_name, free_sms_fragment_limit, sms_rate,
-    #  sms_remainder, sms_billable_units, chargeable_billable_sms, sms_cost)
     assert results[0] == (org.name, org.id, service.name, service.id, 10, Decimal('0.11'), 8, 3, 0, Decimal('0'))
     assert results[1] == (org_2.name, org_2.id, service_2.name, service_2.id, 10, Decimal('0.11'), 0, 3, 3,
                           Decimal('0.33'))
@@ -592,7 +593,8 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
 
 
 def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(notify_db_session):
-    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1))
+    org, org_2, service, service_2, service_3, service_sms_only, \
+        org_with_emails, service_with_emails = set_up_usage_data(datetime(2019, 5, 1))
     results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31))
 
     assert len(results) == 2
@@ -604,7 +606,8 @@ def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(noti
 
 
 def test_fetch_letter_costs_for_all_services(notify_db_session):
-    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1))
+    org, org_2, service, service_2, service_3, service_sms_only, \
+        org_with_emails, service_with_emails = set_up_usage_data(datetime(2019, 6, 1))
 
     results = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
 
@@ -615,7 +618,8 @@ def test_fetch_letter_costs_for_all_services(notify_db_session):
 
 
 def test_fetch_letter_line_items_for_all_service(notify_db_session):
-    org_1, org_2, service_1, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1))
+    org_1, org_2, service_1, service_2, service_3, service_sms_only, \
+        org_with_emails, service_with_emails = set_up_usage_data(datetime(2019, 6, 1))
 
     results = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
 
@@ -625,3 +629,68 @@ def test_fetch_letter_line_items_for_all_service(notify_db_session):
     assert results[2] == (org_2.name, org_2.id, service_2.name, service_2.id, Decimal("0.65"), 'second', 20)
     assert results[3] == (org_2.name, org_2.id, service_2.name, service_2.id, Decimal("0.50"), 'first', 2)
     assert results[4] == (None, None, service_3.name, service_3.id, Decimal("0.55"), 'second', 15)
+
+
+@freeze_time('2019-06-01 13:30')
+def test_fetch_usage_year_for_organisation(notify_db_session):
+    org, org_2, service, service_2, service_3, service_sms_only, \
+        org_with_emails, service_with_emails = set_up_usage_data(datetime(2019, 5, 1))
+    service_with_emails_for_org = create_service(service_name='Service with emails for org')
+    dao_add_service_to_organisation(service=service_with_emails_for_org, organisation_id=org.id)
+    template = create_template(service=service_with_emails_for_org, template_type='email')
+    create_ft_billing(bst_date=datetime(2019, 5, 1),
+                      template=template,
+                      notifications_sent=1100)
+    results = fetch_usage_year_for_organisation(org.id, 2019)
+
+    assert len(results) == 2
+    first_row = results[str(service.id)]
+    assert first_row['service_id'] == service.id
+    assert first_row['service_name'] == service.name
+    assert first_row['free_sms_limit'] == 10
+    assert first_row['sms_remainder'] == 10
+    assert first_row['chargeable_billable_sms'] == 0
+    assert first_row['sms_cost'] == 0.0
+    assert first_row['letter_cost'] == 3.4
+    assert first_row['emails_sent'] == 0
+
+    second_row = results[str(service_with_emails_for_org.id)]
+    assert second_row['service_id'] == service_with_emails_for_org.id
+    assert second_row['service_name'] == service_with_emails_for_org.name
+    assert second_row['free_sms_limit'] == 0
+    assert second_row['sms_remainder'] == 0
+    assert second_row['chargeable_billable_sms'] == 0
+    assert second_row['sms_cost'] == 0
+    assert second_row['letter_cost'] == 0
+    assert second_row['emails_sent'] == 1100
+
+
+def test_fetch_usage_year_for_organisation_populates_ft_billing_for_today(notify_db_session):
+    create_letter_rate(start_date=datetime.utcnow() - timedelta(days=1))
+    create_rate(start_date=datetime.utcnow() - timedelta(days=1), value=0.65, notification_type='sms')
+    new_org = create_organisation(name='New organisation')
+    service = create_service()
+    template = create_template(service=service)
+    dao_add_service_to_organisation(service=service, organisation_id=new_org.id)
+    current_year = datetime.utcnow().year
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=current_year)
+
+    assert FactBilling.query.count() == 0
+
+    create_notification(template=template, status='delivered')
+
+    results = fetch_usage_year_for_organisation(organisation_id=new_org.id, year=current_year)
+    assert len(results) == 1
+    assert FactBilling.query.count() == 1
+
+
+def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(notify_db_session):
+    org = create_organisation(name='Organisation without live services')
+    service = create_service(restricted=True)
+    template = create_template(service=service)
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_ft_billing(bst_date=datetime.utcnow().date(), template=template, billable_unit=19, notifications_sent=19)
+
+    results = fetch_usage_year_for_organisation(organisation_id=org.id, year=datetime.utcnow().year)
+
+    assert len(results) == 0

tests/app/db.py

@@ -906,6 +906,11 @@ def set_up_usage_data(start_date):
     org = create_organisation(name="Org for {}".format(service.name))
     dao_add_service_to_organisation(service=service, organisation_id=org.id)
 
+    service_2 = create_service(service_name='b - emails')
+    email_template = create_template(service=service_2, template_type='email')
+    org_2 = create_organisation(name='Org for {}'.format(service_2.name))
+    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
+
     service_3 = create_service(service_name='c - letters only')
     letter_template_3 = create_template(service=service_3, template_type='letter')
     org_3 = create_organisation(name="Org for {}".format(service_3.name))
@@ -942,7 +947,9 @@ def set_up_usage_data(start_date):
     create_ft_billing(bst_date=two_days_later, template=letter_template_4,
                       notifications_sent=15, billable_unit=4, rate=.55, postage='second')
 
-    return org, org_3, service, service_3, service_4, service_sms_only
+    create_ft_billing(bst_date=start_date, template=email_template, notifications_sent=10)
+
+    return org, org_3, service, service_3, service_4, service_sms_only, org_2, service_2
 
 
 def create_returned_letter(service=None, reported_at=None, notification_id=None):

tests/app/organisation/test_rest.py

@@ -1,6 +1,9 @@
+from datetime import datetime
+
 import uuid
 
 import pytest
+from freezegun import freeze_time
 
 from app.models import Organisation
 from app.dao.organisation_dao import dao_add_service_to_organisation, dao_add_user_to_organisation
@@ -11,6 +14,9 @@ from tests.app.db import (
     create_organisation,
     create_service,
     create_user,
+    create_template,
+    create_ft_billing,
+    create_annual_billing
 )
 
 
@@ -737,3 +743,50 @@ def test_is_organisation_name_unique_returns_400_when_name_does_not_exist(admin_
 
     assert response["message"][0]["org_id"] == ["Can't be empty"]
     assert response["message"][1]["name"] == ["Can't be empty"]
+
+
+@freeze_time('2020-02-24 13:30')
+def test_get_organisation_services_usage(admin_request, notify_db_session):
+    org = create_organisation(name='Organisation without live services')
+    service = create_service()
+    template = create_template(service=service)
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(bst_date=datetime.utcnow().date(), template=template, billable_unit=19, rate=0.060,
+                      notifications_sent=19)
+    response = admin_request.get(
+        'organisation.get_organisation_services_usage',
+        organisation_id=org.id,
+        **{"year": 2019}
+    )
+    assert len(response) == 1
+    assert len(response['services']) == 1
+    service_usage = response['services'][0]
+    assert service_usage['service_id'] == str(service.id)
+    assert service_usage['service_name'] == service.name
+    assert service_usage['chargeable_billable_sms'] == 9.0
+    assert service_usage['emails_sent'] == 0
+    assert service_usage['free_sms_limit'] == 10
+    assert service_usage['letter_cost'] == 0
+    assert service_usage['sms_billable_units'] == 19
+    assert service_usage['sms_remainder'] == 10
+    assert service_usage['sms_cost'] == 0.54
+
+
+def test_get_organisation_services_usage_returns_400_if_year_is_invalid(admin_request):
+    response = admin_request.get(
+        'organisation.get_organisation_services_usage',
+        organisation_id=uuid.uuid4(),
+        **{"year": 'not-a-valid-year'},
+        _expected_status=400
+    )
+    assert response['message'] == 'No valid year provided'
+
+
+def test_get_organisation_services_usage_returns_400_if_year_is_empty(admin_request):
+    response = admin_request.get(
+        'organisation.get_organisation_services_usage',
+        organisation_id=uuid.uuid4(),
+        _expected_status=400
+    )
+    assert response['message'] == 'No valid year provided'

tests/app/platform_stats/test_rest.py

@@ -122,7 +122,8 @@ def test_validate_date_is_within_a_financial_year_when_input_is_not_a_date(start
 
 
 def test_get_usage_for_all_services(notify_db_session, admin_request):
-    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1))
+    org, org_2, service, service_2, service_3, service_sms_only, \
+        org_with_emails, service_with_emails = set_up_usage_data(datetime(2019, 5, 1))
     response = admin_request.get("platform_stats.get_usage_for_all_services",
                                  start_date='2019-05-01',
                                  end_date='2019-06-30')