Merge pull request #273 from alphagov/provider_statistics

Fragment count endpoint added and all tests working.

app/dao/provider_statistics_dao.py

@@ -1,5 +1,40 @@
-from app.models import ProviderStatistics
+from sqlalchemy import func
+from app.models import (ProviderStatistics, SMS_PROVIDERS, EMAIL_PROVIDERS)
 
 
-def get_provider_statistics(service, provider):
+def get_provider_statistics(service, **kwargs):
-    return ProviderStatistics.query.filter_by(service=service, provider=provider).one()
+    return filter_query(ProviderStatistics.query, service, **kwargs)
+
+
+def get_fragment_count(service, date_from, date_to):
+    sms_query = filter_query(
+        ProviderStatistics.query,
+        service,
+        providers=SMS_PROVIDERS,
+        date_from=date_from,
+        date_to=date_to
+    )
+    email_query = filter_query(
+        ProviderStatistics.query,
+        service,
+        providers=EMAIL_PROVIDERS,
+        date_from=date_from,
+        date_to=date_to
+    )
+    return {
+        'sms_count': int(sms_query.with_entities(
+            func.sum(ProviderStatistics.unit_count)).scalar()) if sms_query.count() > 0 else 0,
+        'email_count': int(email_query.with_entities(
+            func.sum(ProviderStatistics.unit_count)).scalar()) if email_query.count() > 0 else 0
+    }
+
+
+def filter_query(query, service, **kwargs):
+    query = query.filter_by(service=service)
+    if 'providers' in kwargs:
+        query = query.filter(ProviderStatistics.provider.in_(kwargs['providers']))
+    if 'date_from' in kwargs:
+        query.filter(ProviderStatistics.day >= kwargs['date_from'])
+    if 'date_to' in kwargs:
+        query.filter(ProviderStatistics.day <= kwargs['date_to'])
+    return query

app/models.py

@@ -187,7 +187,9 @@ TWILIO_PROVIDER = "twilio"
 FIRETEXT_PROVIDER = "firetext"
 SES_PROVIDER = 'ses'
 
-PROVIDERS = [MMG_PROVIDER, TWILIO_PROVIDER, FIRETEXT_PROVIDER, SES_PROVIDER]
+SMS_PROVIDERS = [MMG_PROVIDER, TWILIO_PROVIDER, FIRETEXT_PROVIDER]
+EMAIL_PROVIDERS = [SES_PROVIDER]
+PROVIDERS = SMS_PROVIDERS + EMAIL_PROVIDERS
 
 
 class ProviderStatistics(db.Model):

app/schemas.py

@@ -1,3 +1,4 @@
+from datetime import date
 from flask_marshmallow.fields import fields
 
 from marshmallow import (
@@ -331,6 +332,31 @@ class EventSchema(BaseSchema):
         model = models.Event
 
 
+class FromToDateSchema(ma.Schema):
+
+    date_from = fields.Date()
+    date_to = fields.Date()
+
+    def _validate_not_in_future(self, dte):
+        if dte > date.today():
+            raise ValidationError('Date cannot be in the future')
+
+    @validates('date_from')
+    def validate_date_from(self, value):
+        self._validate_not_in_future(value)
+
+    @validates('date_to')
+    def validate_date_to(self, value):
+        self._validate_not_in_future(value)
+
+    @validates_schema
+    def validate_dates(self, data):
+        df = data.get('date_from')
+        dt = data.get('date_to')
+        if (df and dt) and (df > dt):
+            raise ValidationError("date_from needs to be greater than date_to")
+
+
 user_schema = UserSchema()
 user_schema_load_json = UserSchema(load_json=True)
 service_schema = ServiceSchema()
@@ -359,3 +385,4 @@ service_history_schema = ServiceHistorySchema()
 api_key_history_schema = ApiKeyHistorySchema()
 template_history_schema = TemplateHistorySchema()
 event_schema = EventSchema()
+from_to_date_schema = FromToDateSchema()

app/service/rest.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import (datetime, date)
 
 from flask import Blueprint
 from flask import (
@@ -23,12 +23,15 @@ from app.dao.services_dao import (
     dao_remove_user_from_service
 )
 
+from app.dao.provider_statistics_dao import get_fragment_count
+
 from app.dao.users_dao import get_model_users
 from app.models import ApiKey
 from app.schemas import (
     service_schema,
     api_key_schema,
-    user_schema
+    user_schema,
+    from_to_date_schema
 )
 
 from app.errors import register_errors
@@ -181,6 +184,20 @@ def _process_permissions(user, service, permission_groups):
     return permissions
 
 
+@service.route('/<uuid:service_id>/fragment/aggregate_statistics')
+def get_service_provider_aggregate_statistics(service_id):
+    service = dao_fetch_service_by_id(service_id)
+    data, errors = from_to_date_schema.load(request.args)
+    if errors:
+        return jsonify(result='error', message=errors), 400
+
+    return jsonify(data=get_fragment_count(
+        service,
+        date_from=(data.pop('date_from') if 'date_from' in data else date.today()),
+        date_to=(data.pop('date_to') if 'date_to' in data else date.today())
+    ))
+
+
 # This is placeholder get method until more thought
 # goes into how we want to fetch and view various items in history
 # tables. This is so product owner can pass stories as done

tests/app/conftest.py

@@ -1,8 +1,19 @@
 import pytest
-from datetime import datetime
+from datetime import (datetime, date)
 from app import db
 from app.models import (
-    User, Service, Template, ApiKey, Job, Notification, InvitedUser, Permission)
+    User,
+    Service,
+    Template,
+    ApiKey,
+    Job,
+    Notification,
+    InvitedUser,
+    Permission,
+    MMG_PROVIDER,
+    SES_PROVIDER,
+    TWILIO_PROVIDER,
+    ProviderStatistics)
 from app.dao.users_dao import (save_model_user, create_user_code, create_secret_code)
 from app.dao.services_dao import (dao_create_service, dao_add_user_to_service)
 from app.dao.templates_dao import dao_create_template
@@ -448,9 +459,35 @@ def fake_uuid():
 
 @pytest.fixture(scope='function')
 def ses_provider_name():
-    return 'ses'
+    return SES_PROVIDER
 
 
 @pytest.fixture(scope='function')
 def mmg_provider_name():
-    return 'mmg'
+    return MMG_PROVIDER
+
+
+@pytest.fixture(scope='function')
+def twilio_provider_name():
+    return TWILIO_PROVIDER
+
+
+@pytest.fixture(scope='function')
+def sample_provider_statistics(notify_db,
+                               notify_db_session,
+                               sample_service,
+                               provider=None,
+                               day=None,
+                               unit_count=1):
+    if provider is None:
+        provider = mmg_provider_name()
+    if day is None:
+        day = date.today()
+    stats = ProviderStatistics(
+        service=sample_service,
+        provider=provider,
+        day=day,
+        unit_count=unit_count)
+    notify_db.session.add(stats)
+    notify_db.session.commit()
+    return stats

tests/app/dao/test_notifications_dao_provider_statistics.py

@@ -1,7 +1,8 @@
-from datetime import datetime
+from datetime import (date, timedelta)
 from app.models import ProviderStatistics
 
-from app.dao.provider_statistics_dao import get_provider_statistics
+from app.dao.provider_statistics_dao import (
+    get_provider_statistics, get_fragment_count)
 from app.models import Notification
 from tests.app.conftest import sample_notification as create_sample_notification
 
@@ -14,7 +15,9 @@ def test_should_update_provider_statistics_sms(notify_db,
         notify_db,
         notify_db_session,
         template=sample_template)
-    provider_stats = get_provider_statistics(sample_template.service, mmg_provider_name)
+    provider_stats = get_provider_statistics(
+        sample_template.service,
+        providers=[mmg_provider_name]).one()
     assert provider_stats.unit_count == 1
 
 
@@ -26,7 +29,9 @@ def test_should_update_provider_statistics_email(notify_db,
         notify_db,
         notify_db_session,
         template=sample_email_template)
-    provider_stats = get_provider_statistics(sample_email_template.service, ses_provider_name)
+    provider_stats = get_provider_statistics(
+        sample_email_template.service,
+        providers=[ses_provider_name]).one()
     assert provider_stats.unit_count == 1
 
 
@@ -49,7 +54,9 @@ def test_should_update_provider_statistics_sms_multi(notify_db,
         notify_db_session,
         template=sample_template,
         content_char_count=307)
-    provider_stats = get_provider_statistics(sample_template.service, mmg_provider_name)
+    provider_stats = get_provider_statistics(
+        sample_template.service,
+        providers=[mmg_provider_name]).one()
     assert provider_stats.unit_count == 6
 
 
@@ -69,5 +76,66 @@ def test_should_update_provider_statistics_email_multi(notify_db,
         notify_db,
         notify_db_session,
         template=sample_email_template)
-    provider_stats = get_provider_statistics(sample_email_template.service, ses_provider_name)
+    provider_stats = get_provider_statistics(
+        sample_email_template.service,
+        providers=[ses_provider_name]).one()
     assert provider_stats.unit_count == 3
+
+
+def test_should_aggregate_fragment_count(notify_db,
+                                         notify_db_session,
+                                         sample_service,
+                                         mmg_provider_name,
+                                         twilio_provider_name,
+                                         ses_provider_name):
+    day = date.today()
+    stats_mmg = ProviderStatistics(
+        service=sample_service,
+        day=day,
+        provider=mmg_provider_name,
+        unit_count=2
+    )
+    stats_twilio = ProviderStatistics(
+        service=sample_service,
+        day=day,
+        provider=twilio_provider_name,
+        unit_count=3
+    )
+    stats_twilio = ProviderStatistics(
+        service=sample_service,
+        day=day,
+        provider=ses_provider_name,
+        unit_count=1
+    )
+    notify_db.session.add(stats_mmg)
+    notify_db.session.add(stats_twilio)
+    notify_db.session.commit()
+    results = get_fragment_count(sample_service, day, day)
+    assert results['sms_count'] == 5
+    assert results['email_count'] == 1
+
+
+def test_should_aggregate_fragment_count_over_days(notify_db,
+                                                   notify_db_session,
+                                                   sample_service,
+                                                   mmg_provider_name):
+    today = date.today()
+    yesterday = today - timedelta(days=1)
+    stats_today = ProviderStatistics(
+        service=sample_service,
+        day=today,
+        provider=mmg_provider_name,
+        unit_count=2
+    )
+    stats_yesterday = ProviderStatistics(
+        service=sample_service,
+        day=yesterday,
+        provider=mmg_provider_name,
+        unit_count=3
+    )
+    notify_db.session.add(stats_today)
+    notify_db.session.add(stats_yesterday)
+    notify_db.session.commit()
+    results = get_fragment_count(sample_service, yesterday, today)
+    assert results['sms_count'] == 5
+    assert results['email_count'] == 0

tests/app/service/test_service_fragment_count.py

@@ -0,0 +1,89 @@
+import json
+from datetime import (date, timedelta)
+from flask import url_for
+from tests import create_authorization_header
+
+
+def test_fragment_count(notify_api, sample_provider_statistics):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            endpoint = url_for(
+                'service.get_service_provider_aggregate_statistics',
+                service_id=str(sample_provider_statistics.service.id))
+            auth_header = create_authorization_header(
+                path=endpoint,
+                method='GET'
+            )
+            resp = client.get(
+                endpoint,
+                headers=[auth_header]
+            )
+            assert resp.status_code == 200
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert json_resp['data']['sms_count'] == 1
+
+
+def test_fragment_count_from_to(notify_api, sample_provider_statistics):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            today_str = date.today().strftime('%Y-%m-%d')
+            endpoint = url_for(
+                'service.get_service_provider_aggregate_statistics',
+                service_id=str(sample_provider_statistics.service.id),
+                date_from=today_str,
+                date_to=today_str)
+            auth_header = create_authorization_header(
+                path=endpoint,
+                method='GET'
+            )
+            resp = client.get(
+                endpoint,
+                headers=[auth_header]
+            )
+            assert resp.status_code == 200
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert json_resp['data']['sms_count'] == 1
+
+
+def test_fragment_count_from_greater_than_to(notify_api, sample_provider_statistics):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            today_str = date.today().strftime('%Y-%m-%d')
+            yesterday_str = date.today() - timedelta(days=1)
+            endpoint = url_for(
+                'service.get_service_provider_aggregate_statistics',
+                service_id=str(sample_provider_statistics.service.id),
+                date_from=today_str,
+                date_to=yesterday_str)
+            auth_header = create_authorization_header(
+                path=endpoint,
+                method='GET'
+            )
+            resp = client.get(
+                endpoint,
+                headers=[auth_header]
+            )
+            assert resp.status_code == 400
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert 'date_from needs to be greater than date_to' in json_resp['message']['_schema']
+
+
+def test_fragment_count_in_future(notify_api, sample_provider_statistics):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            tomorrow_str = (date.today() + timedelta(days=1)).strftime('%Y-%m-%d')
+            endpoint = url_for(
+                'service.get_service_provider_aggregate_statistics',
+                service_id=str(sample_provider_statistics.service.id),
+                date_from=tomorrow_str)
+            auth_header = create_authorization_header(
+                path=endpoint,
+                method='GET'
+            )
+            resp = client.get(
+                endpoint,
+                headers=[auth_header]
+            )
+            assert resp.status_code == 400
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert 'Date cannot be in the future' in json_resp['message']['date_from']