darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-01 15:46:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #287 from alphagov/statistics_group_by_week_backup

Browse Source 
Group by Notification Statistics added and all tests working.
This commit is contained in:
NIcholas Staples
2016-05-09 10:17:37 +01:00
parent 448d1058b8 f465066a69
commit f06aaa924e
7 changed files with 375 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
app/dao/notifications_dao.py
View File

@@ -1,5 +1,6 @@
import math
from sqlalchemy import desc, func
from sqlalchemy import (desc, func, Integer)
from sqlalchemy.sql.expression import cast
from datetime import (
datetime,
@@ -51,6 +52,33 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day):
).order_by(desc(NotificationStatistics.day)).first()
def dao_get_7_day_agg_notification_statistics_for_service(service_id,
date_from,
week_count=52):
doy = date_from.timetuple().tm_yday
return db.session.query(
cast(func.floor((func.extract('doy', NotificationStatistics.day) - doy) / 7), Integer),
cast(func.sum(NotificationStatistics.emails_requested), Integer),
cast(func.sum(NotificationStatistics.emails_delivered), Integer),
cast(func.sum(NotificationStatistics.emails_failed), Integer),
cast(func.sum(NotificationStatistics.sms_requested), Integer),
cast(func.sum(NotificationStatistics.sms_delivered), Integer),
cast(func.sum(NotificationStatistics.sms_failed), Integer)
).filter(
NotificationStatistics.service_id == service_id
).filter(
NotificationStatistics.day >= date_from
).filter(
NotificationStatistics.day < date_from + timedelta(days=7 * week_count)
).group_by(
func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7))
).order_by(
desc(func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7)))
).limit(
week_count
)
def dao_get_template_statistics_for_service(service_id, limit_days=None):
filter = [TemplateStatistics.service_id == service_id]
if limit_days is not None:

40
app/notifications_statistics/rest.py
View File

@@ -1,11 +1,18 @@
from datetime import (date, timedelta)
from flask import (
Blueprint,
jsonify,
request
)
from app.dao.notifications_dao import dao_get_notification_statistics_for_service
from app.schemas import notifications_statistics_schema
from app.dao.notifications_dao import (
dao_get_notification_statistics_for_service,
dao_get_7_day_agg_notification_statistics_for_service
)
from app.schemas import (
notifications_statistics_schema,
week_aggregate_notification_statistics_schema
)
notifications_statistics = Blueprint(
'notifications-statistics',
@@ -35,3 +42,32 @@ def get_all_notification_statistics_for_service(service_id):
data, errors = notifications_statistics_schema.dump(statistics, many=True)
return jsonify(data=data)
@notifications_statistics.route('/seven_day_aggregate')
def get_notification_statistics_for_service_seven_day_aggregate(service_id):
data, errors = week_aggregate_notification_statistics_schema.load(request.args)
if errors:
return jsonify(result='error', message=errors), 400
date_from = data['date_from'] if 'date_from' in data else date(date.today().year, 4, 1)
week_count = data['week_count'] if 'week_count' in data else 52
stats = dao_get_7_day_agg_notification_statistics_for_service(
service_id,
date_from,
week_count).all()
json_stats = []
for x in range(week_count - 1, -1, -1):
week_stats = stats.pop(0) if len(stats) > 0 and stats[0][0] == x else [x, 0, 0, 0, 0, 0, 0]
week_start = (date_from + timedelta(days=week_stats[0] * 7))
if week_start <= date.today():
json_stats.append({
'week_start': week_start.strftime('%Y-%m-%d'),
'week_end': (date_from + timedelta(days=(week_stats[0] * 7) + 6)).strftime('%Y-%m-%d'),
'emails_requested': week_stats[1],
'emails_delivered': week_stats[2],
'emails_failed': week_stats[3],
'sms_requested': week_stats[4],
'sms_delivered': week_stats[5],
'sms_failed': week_stats[6]
})
return jsonify(data=json_stats)

49
app/schemas.py
View File

@@ -24,6 +24,20 @@ from app import models
from app.dao.permissions_dao import permission_dao
def _validate_positive_number(value, msg="Not a positive integer"):
try:
page_int = int(value)
if page_int < 1:
raise ValidationError(msg)
except:
raise ValidationError(msg)
def _validate_not_in_future(dte, msg="Date cannot be in the future"):
if dte > date.today():
raise ValidationError(msg)
# TODO I think marshmallow provides a better integration and error handling.
# Would be better to replace functionality in dao with the marshmallow supported
# functionality.
@@ -265,21 +279,13 @@ class NotificationsFilterSchema(ma.Schema):
in_data['status'] = [x.status for x in in_data['status']]
return in_data
def _validate_positive_number(self, value):
try:
page_int = int(value)
if page_int < 1:
raise ValidationError("Not a positive integer")
except:
raise ValidationError("Not a positive integer")
@validates('page')
def validate_page(self, value):
self._validate_positive_number(value)
_validate_positive_number(value)
@validates('page_size')
def validate_page_size(self, value):
self._validate_positive_number(value)
_validate_positive_number(value)
class TemplateStatisticsSchema(BaseSchema):
@@ -323,17 +329,13 @@ class FromToDateSchema(ma.Schema):
date_from = fields.Date()
date_to = fields.Date()
def _validate_not_in_future(self, dte):
if dte > date.today():
raise ValidationError('Date cannot be in the future')
@validates('date_from')
def validate_date_from(self, value):
self._validate_not_in_future(value)
_validate_not_in_future(value)
@validates('date_to')
def validate_date_to(self, value):
self._validate_not_in_future(value)
_validate_not_in_future(value)
@validates_schema
def validate_dates(self, data):
@@ -343,6 +345,20 @@ class FromToDateSchema(ma.Schema):
raise ValidationError("date_from needs to be greater than date_to")
class WeekAggregateNotificationStatisticsSchema(ma.Schema):
date_from = fields.Date()
week_count = fields.Int()
@validates('date_from')
def validate_date_from(self, value):
_validate_not_in_future(value)
@validates('week_count')
def validate_week_count(self, value):
_validate_positive_number(value)
user_schema = UserSchema()
user_schema_load_json = UserSchema(load_json=True)
service_schema = ServiceSchema()
@@ -372,3 +388,4 @@ api_key_history_schema = ApiKeyHistorySchema()
template_history_schema = TemplateHistorySchema()
event_schema = EventSchema()
from_to_date_schema = FromToDateSchema()
week_aggregate_notification_statistics_schema = WeekAggregateNotificationStatisticsSchema()

32
tests/app/conftest.py
View File

@@ -13,7 +13,8 @@ from app.models import (
MMG_PROVIDER,
SES_PROVIDER,
TWILIO_PROVIDER,
ProviderStatistics)
ProviderStatistics,
NotificationStatistics)
from app.dao.users_dao import (save_model_user, create_user_code, create_secret_code)
from app.dao.services_dao import (dao_create_service, dao_add_user_to_service)
from app.dao.templates_dao import dao_create_template
@@ -493,3 +494,32 @@ def sample_provider_statistics(notify_db,
notify_db.session.add(stats)
notify_db.session.commit()
return stats
@pytest.fixture(scope='function')
def sample_notification_statistics(notify_db,
notify_db_session,
service=None,
day=None,
emails_requested=2,
emails_delivered=1,
emails_failed=1,
sms_requested=2,
sms_delivered=1,
sms_failed=1):
if service is None:
service = sample_service(notify_db, notify_db_session)
if day is None:
day = date.today()
stats = NotificationStatistics(
service=service,
day=day,
emails_requested=emails_requested,
emails_delivered=emails_delivered,
emails_failed=emails_failed,
sms_requested=sms_requested,
sms_delivered=sms_delivered,
sms_failed=sms_failed)
notify_db.session.add(stats)
notify_db.session.commit()
return stats

122
tests/app/dao/test_notification_statistics_dao.py Normal file
View File

@@ -0,0 +1,122 @@
from datetime import (date, timedelta)
from app.models import NotificationStatistics
from tests.app.conftest import sample_notification_statistics as create_sample_notification_statistics
from app.dao.notifications_dao import dao_get_7_day_agg_notification_statistics_for_service
def test_display_weekly_notification_statistics_sum_over_week(notify_db,
notify_db_session,
sample_service):
fools = date(2016, 4, 1)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools + timedelta(days=1)
)
assert dao_get_7_day_agg_notification_statistics_for_service(
sample_service.id,
fools
).all() == [(0, 4, 2, 2, 4, 2, 2)]
def test_display_weekly_notification_statistics_separate_over_weeks(notify_db,
notify_db_session,
sample_service):
fools = date(2016, 4, 1)
next_week = fools + timedelta(days=7)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=next_week
)
assert dao_get_7_day_agg_notification_statistics_for_service(
sample_service.id,
fools
).all() == [(1, 2, 1, 1, 2, 1, 1), (0, 2, 1, 1, 2, 1, 1)]
def test_display_weekly_notification_statistics_7_days_from_date_from(notify_db,
notify_db_session,
sample_service):
fools = date(2016, 4, 1)
eow_fools = fools + timedelta(days=6)
next_week = fools + timedelta(days=7)
two_weeks_later = fools + timedelta(days=14)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=eow_fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=next_week
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=two_weeks_later
)
assert dao_get_7_day_agg_notification_statistics_for_service(
sample_service.id,
fools
).all() == [(2, 2, 1, 1, 2, 1, 1), (1, 2, 1, 1, 2, 1, 1), (0, 4, 2, 2, 4, 2, 2)]
def test_display_weekly_notification_statistics_week_number_misses_week(notify_db,
notify_db_session,
sample_service):
fools = date(2016, 4, 1)
two_weeks_later = fools + timedelta(days=14)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=two_weeks_later
)
assert dao_get_7_day_agg_notification_statistics_for_service(
sample_service.id,
fools
).all() == [(2, 2, 1, 1, 2, 1, 1), (0, 2, 1, 1, 2, 1, 1)]
def test_display_weekly_notification_statistics_week_limit(notify_db,
notify_db_session,
sample_service):
fools = date(2016, 4, 1)
two_weeks_later = fools + timedelta(days=14)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=fools
)
create_sample_notification_statistics(
notify_db,
notify_db_session,
day=two_weeks_later
)
assert dao_get_7_day_agg_notification_statistics_for_service(
sample_service.id,
fools,
1
).all() == [(0, 2, 1, 1, 2, 1, 1)]

122
tests/app/notifications/test_notification_statistics_rest.py Normal file
View File

@@ -0,0 +1,122 @@
import json
from datetime import (date, timedelta)
from flask import url_for
from tests import create_authorization_header
from tests.app.conftest import sample_notification_statistics as create_sample_notification_statistics
def test_get_notification_statistics(
notify_api,
notify_db,
notify_db_session,
sample_template,
sample_email_template
):
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/notifications-statistics'.format(sample_email_template.service)
auth_header = create_authorization_header(
service_id=sample_email_template.service_id)
response = client.get(path, headers=[auth_header])
assert response.status_code == 404
stats = json.loads(response.get_data(as_text=True))
assert stats['result'] == 'error'
assert stats['message'] == 'No result found'
def test_get_week_aggregate_statistics(notify_api,
notify_db,
notify_db_session,
sample_service):
with notify_api.test_request_context():
sample_notification_statistics = create_sample_notification_statistics(
notify_db,
notify_db_session,
day=date(date.today().year, 4, 1))
with notify_api.test_client() as client:
endpoint = url_for(
'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate',
service_id=sample_service.id)
auth_header = create_authorization_header(
service_id=sample_service.id)
resp = client.get(endpoint, headers=[auth_header])
assert resp.status_code == 200
json_resp = json.loads(resp.get_data(as_text=True))
week_len_index = len(json_resp['data']) - 1
assert json_resp['data'][week_len_index]['emails_requested'] == 2
assert json_resp['data'][week_len_index]['sms_requested'] == 2
assert json_resp['data'][week_len_index]['week_start'] == date(date.today().year, 4, 1).strftime('%Y-%m-%d')
assert json_resp['data'][week_len_index]['week_end'] == date(date.today().year, 4, 7).strftime('%Y-%m-%d')
def test_get_week_aggregate_statistics_date_from(notify_api,
notify_db,
notify_db_session,
sample_service):
with notify_api.test_request_context():
sample_notification_statistics = create_sample_notification_statistics(
notify_db,
notify_db_session,
day=date(date.today().year, 4, 1))
date_from_str = date(date.today().year, 4, 1).strftime('%Y-%m-%d')
with notify_api.test_client() as client:
endpoint = url_for(
'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate',
service_id=sample_service.id,
date_from=date_from_str)
auth_header = create_authorization_header(
service_id=sample_service.id)
resp = client.get(endpoint, headers=[auth_header])
assert resp.status_code == 200
json_resp = json.loads(resp.get_data(as_text=True))
week_len_index = len(json_resp['data']) - 1
assert json_resp['data'][week_len_index]['emails_requested'] == 2
assert json_resp['data'][week_len_index]['sms_requested'] == 2
assert json_resp['data'][week_len_index]['week_start'] == date_from_str
assert json_resp['data'][week_len_index]['week_end'] == date(date.today().year, 4, 7).strftime('%Y-%m-%d')
def test_get_week_aggregate_statistics_date_in_future(notify_api,
notify_db,
notify_db_session,
sample_service):
with notify_api.test_request_context():
with notify_api.test_client() as client:
endpoint = url_for(
'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate',
service_id=sample_service.id,
date_from=(date.today() + timedelta(days=1)).strftime('%Y-%m-%d'))
auth_header = create_authorization_header(
service_id=sample_service.id)
resp = client.get(endpoint, headers=[auth_header])
assert resp.status_code == 400
json_resp = json.loads(resp.get_data(as_text=True))
assert json_resp['result'] == 'error'
assert json_resp['message']['date_from'][0] == 'Date cannot be in the future'
def test_get_week_aggregate_statistics_invalid_week_count(notify_api,
notify_db,
notify_db_session,
sample_service):
with notify_api.test_request_context():
with notify_api.test_client() as client:
endpoint = url_for(
'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate',
service_id=sample_service.id,
week_count=-1)
auth_header = create_authorization_header(
service_id=sample_service.id)
resp = client.get(endpoint, headers=[auth_header])
assert resp.status_code == 400
json_resp = json.loads(resp.get_data(as_text=True))
assert json_resp['result'] == 'error'
assert json_resp['message']['week_count'][0] == 'Not a positive integer'

22
tests/app/notifications/test_rest.py
View File

@@ -442,28 +442,6 @@ def test_filter_by_status_and_template_type(notify_api,
assert notifications['notifications'][0]['status'] == 'delivered'
def test_get_notification_statistics(
notify_api,
notify_db,
notify_db_session,
sample_template,
sample_email_template
):
with notify_api.test_request_context():
with notify_api.test_client() as client:
path = '/service/{}/notifications-statistics'.format(sample_email_template.service)
auth_header = create_authorization_header(service_id=sample_email_template.service_id)
response = client.get(path, headers=[auth_header])
assert response.status_code == 404
stats = json.loads(response.get_data(as_text=True))
assert stats['result'] == 'error'
assert stats['message'] == 'No result found'
def test_create_sms_should_reject_if_missing_required_fields(notify_api, sample_api_key, mocker):
with notify_api.test_request_context():
with notify_api.test_client() as client: