Merge pull request #2483 from alphagov/serialize-sent-stats

Add get_live_services_data endpoint and related query

app/dao/date_util.py

@@ -7,8 +7,8 @@ import pytz
 def get_months_for_financial_year(year):
     return [
         convert_bst_to_utc(month) for month in (
-            get_months_for_year(4, 13, year) +
+            get_months_for_year(4, 13, year)
-            get_months_for_year(1, 4, year + 1)
+            + get_months_for_year(1, 4, year + 1)
         )
         if convert_bst_to_utc(month) < datetime.now()
     ]
@@ -22,6 +22,14 @@ def get_financial_year(year):
     return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
 
 
+def get_current_financial_year():
+    now = datetime.utcnow()
+    current_month = int(now.strftime('%-m'))
+    current_year = int(now.strftime('%Y'))
+    year = current_year if current_month > 3 else current_year - 1
+    return get_financial_year(year)
+
+
 def get_april_fools(year):
     """
      This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,

app/dao/services_dao.py

@@ -2,11 +2,12 @@ import uuid
 from datetime import date, datetime, timedelta
 
 from notifications_utils.statsd_decorators import statsd
-from sqlalchemy import asc, func
+from sqlalchemy import asc, func, case
 from sqlalchemy.orm import joinedload
 from flask import current_app
 
 from app import db
+from app.dao.date_util import get_current_financial_year
 from app.dao.dao_utils import (
     transactional,
     version_class,
@@ -21,11 +22,13 @@ from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
     AnnualBilling,
     ApiKey,
+    FactBilling,
     InboundNumber,
     InvitedUser,
     Job,
     Notification,
     NotificationHistory,
+    Organisation,
     Permission,
     Service,
     ServicePermission,
@@ -72,6 +75,95 @@ def dao_count_live_services():
     ).count()
 
 
+def dao_fetch_live_services_data():
+    year_start_date, year_end_date = get_current_financial_year()
+    this_year_ft_billing = FactBilling.query.filter(
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+    ).subquery()
+    data = db.session.query(
+        Service.id,
+        Organisation.name.label("organisation_name"),
+        Service.name.label("service_name"),
+        Service.consent_to_research,
+        Service.go_live_user_id,
+        Service.count_as_live,
+        User.name.label('user_name'),
+        User.email_address,
+        User.mobile_number,
+        Service.go_live_at.label("live_date"),
+        Service.volume_sms,
+        Service.volume_email,
+        Service.volume_letter,
+        case([
+            (this_year_ft_billing.c.notification_type == 'email', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("email_totals"),
+        case([
+            (this_year_ft_billing.c.notification_type == 'sms', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("sms_totals"),
+        case([
+            (this_year_ft_billing.c.notification_type == 'letter', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("letter_totals"),
+    ).outerjoin(
+        Service.organisation
+    ).outerjoin(
+        this_year_ft_billing, Service.id == this_year_ft_billing.c.service_id
+    ).outerjoin(
+        User, Service.go_live_user_id == User.id
+    ).filter(
+        Service.count_as_live == True  # noqa
+    ).group_by(
+        Service.id,
+        Organisation.name,
+        Service.name,
+        Service.consent_to_research,
+        Service.count_as_live,
+        Service.go_live_user_id,
+        User.name,
+        User.email_address,
+        User.mobile_number,
+        Service.go_live_at,
+        Service.volume_sms,
+        Service.volume_email,
+        Service.volume_letter,
+        this_year_ft_billing.c.notification_type
+    ).order_by(
+        asc(Service.go_live_at)
+    ).all()
+    results = []
+    for row in data:
+        is_service_in_list = None
+        i = 0
+        while i < len(results):
+            if results[i]["service_id"] == row.id:
+                is_service_in_list = i
+                break
+            else:
+                i += 1
+        if is_service_in_list is not None:
+            results[is_service_in_list]["email_totals"] += row.email_totals
+            results[is_service_in_list]["sms_totals"] += row.sms_totals
+            results[is_service_in_list]["letter_totals"] += row.letter_totals
+        else:
+            results.append({
+                "service_id": row.id,
+                "service_name": row.service_name,
+                "organisation_name": row.organisation_name,
+                "consent_to_research": row.consent_to_research,
+                "contact_name": row.user_name,
+                "contact_email": row.email_address,
+                "contact_mobile": row.mobile_number,
+                "live_date": row.live_date,
+                "sms_volume_intent": row.volume_sms,
+                "email_volume_intent": row.volume_email,
+                "letter_volume_intent": row.volume_letter,
+                "sms_totals": row.sms_totals,
+                "email_totals": row.email_totals,
+                "letter_totals": row.letter_totals,
+            })
+    return results
+
+
 def dao_fetch_service_by_id(service_id, only_active=False):
     query = Service.query.filter_by(
         id=service_id

app/service/rest.py

@@ -49,6 +49,7 @@ from app.dao.services_dao import (
     dao_create_service,
     dao_fetch_all_services,
     dao_fetch_all_services_by_user,
+    dao_fetch_live_services_data,
     dao_fetch_service_by_id,
     dao_fetch_todays_stats_for_service,
     dao_fetch_todays_stats_for_all_services,
@@ -158,6 +159,12 @@ def get_services():
     return jsonify(data=data)
 
 
+@service_blueprint.route('/live-services-data', methods=['GET'])
+def get_live_services_data():
+    data = dao_fetch_live_services_data()
+    return jsonify(data=data)
+
+
 @service_blueprint.route('/<uuid:service_id>', methods=['GET'])
 def get_service_by_id(service_id):
     if request.args.get('detailed') == 'True':

tests/app/dao/test_services_dao.py

@@ -12,12 +12,14 @@ from app.dao.inbound_numbers_dao import (
     dao_get_available_inbound_numbers,
     dao_set_inbound_number_active_flag
 )
+from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission
 from app.dao.services_dao import (
     dao_create_service,
     dao_add_user_to_service,
     dao_remove_user_from_service,
     dao_fetch_all_services,
+    dao_fetch_live_services_data,
     dao_fetch_service_by_id,
     dao_fetch_all_services_by_user,
     dao_update_service,
@@ -57,7 +59,9 @@ from app.models import (
     user_folder_permissions,
 )
 from tests.app.db import (
+    create_ft_billing,
     create_inbound_number,
+    create_organisation,
     create_user,
     create_service,
     create_service_with_inbound_number,
@@ -380,6 +384,52 @@ def test_get_all_user_services_should_return_empty_list_if_no_services_for_user(
     assert len(dao_fetch_all_services_by_user(user.id)) == 0
 
 
+@freeze_time('2019-04-23T10:00:00')
+def test_dao_fetch_live_services_data(sample_user, mock):
+    org = create_organisation()
+    service = create_service(go_live_user=sample_user, go_live_at='2014-04-20T10:00:00')
+    template = create_template(service=service)
+    service_2 = create_service(service_name='second', go_live_user=sample_user, go_live_at='2017-04-20T10:00:00')
+    create_service(service_name='third', go_live_at='2016-04-20T10:00:00')
+    create_service(service_name='not_live', count_as_live=False)
+    template2 = create_template(service=service, template_type='email')
+    template_letter_1 = create_template(service=service, template_type='letter')
+    template_letter_2 = create_template(service=service_2, template_type='letter')
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    # two sms billing records for 1st service within current financial year:
+    create_ft_billing(bst_date='2019-04-20', notification_type='sms', template=template, service=service)
+    create_ft_billing(bst_date='2019-04-21', notification_type='sms', template=template, service=service)
+    # one sms billing record for 1st service from previous financial year, should not appear in the result:
+    create_ft_billing(bst_date='2018-04-20', notification_type='sms', template=template, service=service)
+    # one email billing record for 1st service within current financial year:
+    create_ft_billing(bst_date='2019-04-20', notification_type='email', template=template2, service=service)
+    # one letter billing record for 1st service within current financial year:
+    create_ft_billing(bst_date='2019-04-15', notification_type='letter', template=template_letter_1, service=service)
+    # one letter billing record for 2nd service within current financial year:
+    create_ft_billing(bst_date='2019-04-16', notification_type='letter', template=template_letter_2, service=service_2)
+
+    results = dao_fetch_live_services_data()
+    assert len(results) == 3
+    # checks the results and that they are ordered by date:
+    assert results == [
+        {'service_id': mock.ANY, 'service_name': 'Sample service', 'organisation_name': 'test_org_1',
+            'consent_to_research': None, 'contact_name': 'Test User',
+            'contact_email': 'notify@digital.cabinet-office.gov.uk', 'contact_mobile': '+447700900986',
+            'live_date': datetime(2014, 4, 20, 10, 0), 'sms_volume_intent': None, 'email_volume_intent': None,
+            'letter_volume_intent': None, 'sms_totals': 2, 'email_totals': 1, 'letter_totals': 1},
+        {'service_id': mock.ANY, 'service_name': 'third', 'organisation_name': None, 'consent_to_research': None,
+            'contact_name': None, 'contact_email': None,
+            'contact_mobile': None, 'live_date': datetime(2016, 4, 20, 10, 0), 'sms_volume_intent': None,
+            'email_volume_intent': None, 'letter_volume_intent': None,
+            'sms_totals': 0, 'email_totals': 0, 'letter_totals': 0},
+        {'service_id': mock.ANY, 'service_name': 'second', 'organisation_name': None, 'consent_to_research': None,
+            'contact_name': 'Test User', 'contact_email': 'notify@digital.cabinet-office.gov.uk',
+            'contact_mobile': '+447700900986', 'live_date': datetime(2017, 4, 20, 10, 0), 'sms_volume_intent': None,
+            'email_volume_intent': None, 'letter_volume_intent': None,
+            'sms_totals': 0, 'email_totals': 0, 'letter_totals': 1}
+    ]
+
+
 def test_get_service_by_id_returns_none_if_no_service(notify_db):
     with pytest.raises(NoResultFound) as e:
         dao_fetch_service_by_id(str(uuid.uuid4()))

tests/app/service/test_rest.py

@@ -35,6 +35,7 @@ from tests.app.conftest import (
     sample_notification_with_job
 )
 from tests.app.db import (
+    create_ft_billing,
     create_ft_notification_status,
     create_service,
     create_service_with_inbound_number,
@@ -136,6 +137,30 @@ def test_get_service_list_should_return_empty_list_if_no_services(admin_request)
     assert len(json_resp['data']) == 0
 
 
+def test_get_live_services_data(sample_user, admin_request, mock):
+    org = create_organisation()
+    service = create_service(go_live_user=sample_user)
+    template = create_template(service=service)
+    create_service(service_name='second', go_live_user=sample_user)
+    template2 = create_template(service=service, template_type='email')
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_ft_billing(bst_date='2019-04-20', notification_type='sms', template=template, service=service)
+    create_ft_billing(bst_date='2019-04-20', notification_type='email', template=template2,
+                      service=service)
+    response = admin_request.get('service.get_live_services_data')["data"]
+    assert len(response) == 2
+    assert {'consent_to_research': None, 'contact_email': 'notify@digital.cabinet-office.gov.uk',
+            'contact_mobile': '+447700900986', 'contact_name': 'Test User', 'email_totals': 0,
+            'email_volume_intent': None, 'letter_totals': 0, 'letter_volume_intent': None, 'live_date': None,
+            'organisation_name': None, 'service_id': mock.ANY, 'service_name': 'second', 'sms_totals': 0,
+            'sms_volume_intent': None} in response
+    assert {'consent_to_research': None, 'contact_email': 'notify@digital.cabinet-office.gov.uk',
+            'contact_mobile': '+447700900986', 'contact_name': 'Test User', 'email_totals': 1,
+            'email_volume_intent': None, 'letter_totals': 0, 'letter_volume_intent': None, 'live_date': None,
+            'organisation_name': 'test_org_1', 'service_id': mock.ANY, 'service_name': 'Sample service',
+            'sms_totals': 1, 'sms_volume_intent': None} in response
+
+
 def test_get_service_by_id(admin_request, sample_service):
     json_resp = admin_request.get('service.get_service_by_id', service_id=sample_service.id)
     assert json_resp['data']['name'] == sample_service.name