Merge pull request #287 from alphagov/statistics_group_by_week_backup

Group by Notification Statistics added and all tests working.

app/dao/notifications_dao.py

@@ -1,5 +1,6 @@
 import math
-from sqlalchemy import desc, func
+from sqlalchemy import (desc, func, Integer)
+from sqlalchemy.sql.expression import cast
 
 from datetime import (
     datetime,
@@ -51,6 +52,33 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day):
     ).order_by(desc(NotificationStatistics.day)).first()
 
 
+def dao_get_7_day_agg_notification_statistics_for_service(service_id,
+                                                          date_from,
+                                                          week_count=52):
+    doy = date_from.timetuple().tm_yday
+    return db.session.query(
+        cast(func.floor((func.extract('doy', NotificationStatistics.day) - doy) / 7), Integer),
+        cast(func.sum(NotificationStatistics.emails_requested), Integer),
+        cast(func.sum(NotificationStatistics.emails_delivered), Integer),
+        cast(func.sum(NotificationStatistics.emails_failed), Integer),
+        cast(func.sum(NotificationStatistics.sms_requested), Integer),
+        cast(func.sum(NotificationStatistics.sms_delivered), Integer),
+        cast(func.sum(NotificationStatistics.sms_failed), Integer)
+    ).filter(
+        NotificationStatistics.service_id == service_id
+    ).filter(
+        NotificationStatistics.day >= date_from
+    ).filter(
+        NotificationStatistics.day < date_from + timedelta(days=7 * week_count)
+    ).group_by(
+        func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7))
+    ).order_by(
+        desc(func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7)))
+    ).limit(
+        week_count
+    )
+
+
 def dao_get_template_statistics_for_service(service_id, limit_days=None):
     filter = [TemplateStatistics.service_id == service_id]
     if limit_days is not None:

app/notifications_statistics/rest.py

@@ -1,11 +1,18 @@
+from datetime import (date, timedelta)
 from flask import (
     Blueprint,
     jsonify,
     request
 )
 
-from app.dao.notifications_dao import dao_get_notification_statistics_for_service
-from app.schemas import notifications_statistics_schema
+from app.dao.notifications_dao import (
+    dao_get_notification_statistics_for_service,
+    dao_get_7_day_agg_notification_statistics_for_service
+)
+from app.schemas import (
+    notifications_statistics_schema,
+    week_aggregate_notification_statistics_schema
+)
 
 notifications_statistics = Blueprint(
     'notifications-statistics',
@@ -35,3 +42,32 @@ def get_all_notification_statistics_for_service(service_id):
 
     data, errors = notifications_statistics_schema.dump(statistics, many=True)
     return jsonify(data=data)
+
+
+@notifications_statistics.route('/seven_day_aggregate')
+def get_notification_statistics_for_service_seven_day_aggregate(service_id):
+    data, errors = week_aggregate_notification_statistics_schema.load(request.args)
+    if errors:
+        return jsonify(result='error', message=errors), 400
+    date_from = data['date_from'] if 'date_from' in data else date(date.today().year, 4, 1)
+    week_count = data['week_count'] if 'week_count' in data else 52
+    stats = dao_get_7_day_agg_notification_statistics_for_service(
+        service_id,
+        date_from,
+        week_count).all()
+    json_stats = []
+    for x in range(week_count - 1, -1, -1):
+        week_stats = stats.pop(0) if len(stats) > 0 and stats[0][0] == x else [x, 0, 0, 0, 0, 0, 0]
+        week_start = (date_from + timedelta(days=week_stats[0] * 7))
+        if week_start <= date.today():
+            json_stats.append({
+                'week_start': week_start.strftime('%Y-%m-%d'),
+                'week_end': (date_from + timedelta(days=(week_stats[0] * 7) + 6)).strftime('%Y-%m-%d'),
+                'emails_requested': week_stats[1],
+                'emails_delivered': week_stats[2],
+                'emails_failed': week_stats[3],
+                'sms_requested': week_stats[4],
+                'sms_delivered': week_stats[5],
+                'sms_failed': week_stats[6]
+            })
+    return jsonify(data=json_stats)

app/schemas.py

@@ -24,6 +24,20 @@ from app import models
 from app.dao.permissions_dao import permission_dao
 
 
+def _validate_positive_number(value, msg="Not a positive integer"):
+    try:
+        page_int = int(value)
+        if page_int < 1:
+            raise ValidationError(msg)
+    except:
+        raise ValidationError(msg)
+
+
+def _validate_not_in_future(dte, msg="Date cannot be in the future"):
+    if dte > date.today():
+        raise ValidationError(msg)
+
+
 # TODO I think marshmallow provides a better integration and error handling.
 # Would be better to replace functionality in dao with the marshmallow supported
 # functionality.
@@ -265,21 +279,13 @@ class NotificationsFilterSchema(ma.Schema):
             in_data['status'] = [x.status for x in in_data['status']]
         return in_data
 
-    def _validate_positive_number(self, value):
-        try:
-            page_int = int(value)
-            if page_int < 1:
-                raise ValidationError("Not a positive integer")
-        except:
-            raise ValidationError("Not a positive integer")
-
     @validates('page')
     def validate_page(self, value):
-        self._validate_positive_number(value)
+        _validate_positive_number(value)
 
     @validates('page_size')
     def validate_page_size(self, value):
-        self._validate_positive_number(value)
+        _validate_positive_number(value)
 
 
 class TemplateStatisticsSchema(BaseSchema):
@@ -323,17 +329,13 @@ class FromToDateSchema(ma.Schema):
     date_from = fields.Date()
     date_to = fields.Date()
 
-    def _validate_not_in_future(self, dte):
-        if dte > date.today():
-            raise ValidationError('Date cannot be in the future')
-
     @validates('date_from')
     def validate_date_from(self, value):
-        self._validate_not_in_future(value)
+        _validate_not_in_future(value)
 
     @validates('date_to')
     def validate_date_to(self, value):
-        self._validate_not_in_future(value)
+        _validate_not_in_future(value)
 
     @validates_schema
     def validate_dates(self, data):
@@ -343,6 +345,20 @@ class FromToDateSchema(ma.Schema):
             raise ValidationError("date_from needs to be greater than date_to")
 
 
+class WeekAggregateNotificationStatisticsSchema(ma.Schema):
+
+    date_from = fields.Date()
+    week_count = fields.Int()
+
+    @validates('date_from')
+    def validate_date_from(self, value):
+        _validate_not_in_future(value)
+
+    @validates('week_count')
+    def validate_week_count(self, value):
+        _validate_positive_number(value)
+
+
 user_schema = UserSchema()
 user_schema_load_json = UserSchema(load_json=True)
 service_schema = ServiceSchema()
@@ -372,3 +388,4 @@ api_key_history_schema = ApiKeyHistorySchema()
 template_history_schema = TemplateHistorySchema()
 event_schema = EventSchema()
 from_to_date_schema = FromToDateSchema()
+week_aggregate_notification_statistics_schema = WeekAggregateNotificationStatisticsSchema()