Merge branch 'master' into primary-provider

Conflicts:
	app/dao/notifications_dao.py
	app/dao/provider_statistics_dao.py
	app/schemas.py
	tests/app/conftest.py

app/authentication/auth.py

@@ -8,7 +8,6 @@ from functools import wraps
 
 
 def authentication_response(message, code):
-    current_app.logger.info(message)
     return jsonify(result='error',
                    message=message
                    ), code
@@ -36,10 +35,7 @@ def requires_auth():
         try:
             decode_jwt_token(
                 auth_token,
-                secret,
-                request.method,
-                request.path,
-                request.data.decode() if request.data else None
+                secret
             )
             _request_ctx_stack.top.api_user = api_client
             return
@@ -50,6 +46,7 @@ def requires_auth():
 
     if not api_client['secret']:
         errors_resp = authentication_response("Invalid token: signature", 403)
+    current_app.logger.info(errors_resp)
     return errors_resp
 
 

app/commands.py

@@ -1,8 +1,11 @@
+import uuid
 from datetime import datetime
 from decimal import Decimal
 from flask.ext.script import Command, Manager, Option
-from app.models import PROVIDERS
+from app.models import (PROVIDERS, Service, User)
+from app.dao.services_dao import delete_service_and_all_associated_db_objects
 from app.dao.provider_rates_dao import create_provider_rates
+from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
 
 
 class CreateProviderRateCommand(Command):
@@ -28,3 +31,36 @@ class CreateProviderRateCommand(Command):
             raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S")
 
         create_provider_rates(provider_name, valid_from, cost)
+
+
+class PurgeFunctionalTestDataCommand(Command):
+
+    option_list = (
+        Option('-n', '-service-name-prefix', dest="service_name_prefix", help='Functional service name prefix.'),
+        Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix.")
+    )
+
+    def run(self, service_name_prefix=None, user_email_prefix=None):
+        if service_name_prefix:
+            services = Service.query.filter(Service.name.like("{}%".format(service_name_prefix))).all()
+            for service in services:
+                # Make sure the second part of the service name is a uuid.
+                # Just in case someone decides to create a service with that name included in it.
+                try:
+                    uuid.UUID(service.name.split(service_name_prefix)[1])
+                except ValueError:
+                    print("Skipping {} as the service name doesn't contain a UUID.".format(service.name))
+                else:
+                    delete_service_and_all_associated_db_objects(service)
+        if user_email_prefix:
+            users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
+            for usr in users:
+                # Make sure the full email includes a uuid in it
+                # Just in case someone decides to use a similar email address.
+                try:
+                    uuid.UUID(usr.email_address.split("@")[0].split('+')[1])
+                except ValueError:
+                    print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address))
+                else:
+                    delete_user_verify_codes(usr)
+                    delete_model_user(usr)

app/dao/notifications_dao.py

@@ -1,4 +1,5 @@
-from sqlalchemy import desc, func
+from sqlalchemy import (desc, func, Integer)
+from sqlalchemy.sql.expression import cast
 
 from datetime import (
     datetime,
@@ -50,6 +51,33 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day):
     ).order_by(desc(NotificationStatistics.day)).first()
 
 
+def dao_get_7_day_agg_notification_statistics_for_service(service_id,
+                                                          date_from,
+                                                          week_count=52):
+    doy = date_from.timetuple().tm_yday
+    return db.session.query(
+        cast(func.floor((func.extract('doy', NotificationStatistics.day) - doy) / 7), Integer),
+        cast(func.sum(NotificationStatistics.emails_requested), Integer),
+        cast(func.sum(NotificationStatistics.emails_delivered), Integer),
+        cast(func.sum(NotificationStatistics.emails_failed), Integer),
+        cast(func.sum(NotificationStatistics.sms_requested), Integer),
+        cast(func.sum(NotificationStatistics.sms_delivered), Integer),
+        cast(func.sum(NotificationStatistics.sms_failed), Integer)
+    ).filter(
+        NotificationStatistics.service_id == service_id
+    ).filter(
+        NotificationStatistics.day >= date_from
+    ).filter(
+        NotificationStatistics.day < date_from + timedelta(days=7 * week_count)
+    ).group_by(
+        func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7))
+    ).order_by(
+        desc(func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7)))
+    ).limit(
+        week_count
+    )
+
+
 def dao_get_template_statistics_for_service(service_id, limit_days=None):
     filter = [TemplateStatistics.service_id == service_id]
     if limit_days is not None:
@@ -201,11 +229,10 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
         page_size = current_app.config['PAGE_SIZE']
     query = Notification.query.filter_by(service_id=service_id, job_id=job_id)
     query = filter_query(query, filter_dict)
-    pagination = query.order_by(desc(Notification.created_at)).paginate(
+    return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
         per_page=page_size
     )
-    return pagination
 
 
 def get_notification(service_id, notification_id):
@@ -216,7 +243,11 @@ def get_notification_by_id(notification_id):
     return Notification.query.filter_by(id=notification_id).first()
 
 
-def get_notifications_for_service(service_id, filter_dict=None, page=1, page_size=None, limit_days=None):
+def get_notifications_for_service(service_id,
+                                  filter_dict=None,
+                                  page=1,
+                                  page_size=None,
+                                  limit_days=None):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
     filters = [Notification.service_id == service_id]
@@ -227,11 +258,10 @@ def get_notifications_for_service(service_id, filter_dict=None, page=1, page_siz
 
     query = Notification.query.filter(*filters)
     query = filter_query(query, filter_dict)
-    pagination = query.order_by(desc(Notification.created_at)).paginate(
+    return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
         per_page=page_size
     )
-    return pagination
 
 
 def filter_query(query, filter_dict=None):

app/dao/provider_details_dao.py

@@ -3,6 +3,7 @@ from app.dao.dao_utils import transactional
 from app.models import ProviderDetails
 from app import db
 
+
 def get_provider_details():
     return ProviderDetails.query.order_by(asc(ProviderDetails.priority)).all()
 

app/dao/services_dao.py

@@ -9,6 +9,21 @@ from app.dao.dao_utils import (
     version_class
 )
 
+from app.models import (
+    NotificationStatistics,
+    TemplateStatistics,
+    ProviderStatistics,
+    VerifyCode,
+    ApiKey,
+    Template,
+    Job,
+    Notification,
+    Permission,
+    User,
+    InvitedUser,
+    Service
+)
+
 
 def dao_fetch_all_services():
     return Service.query.order_by(asc(Service.created_at)).all()
@@ -66,3 +81,31 @@ def dao_remove_user_from_service(service, user):
         raise e
     else:
         db.session.commit()
+
+
+def delete_service_and_all_associated_db_objects(service):
+
+    def _delete_commit(query):
+        query.delete()
+        db.session.commit()
+
+    _delete_commit(NotificationStatistics.query.filter_by(service=service))
+    _delete_commit(TemplateStatistics.query.filter_by(service=service))
+    _delete_commit(ProviderStatistics.query.filter_by(service=service))
+    _delete_commit(InvitedUser.query.filter_by(service=service))
+    _delete_commit(Permission.query.filter_by(service=service))
+    _delete_commit(ApiKey.query.filter_by(service=service))
+    _delete_commit(Notification.query.filter_by(service=service))
+    _delete_commit(Job.query.filter_by(service=service))
+    _delete_commit(Template.query.filter_by(service=service))
+
+    verify_codes = VerifyCode.query.join(User).filter(User.id.in_([x.id for x in service.users]))
+    list(map(db.session.delete, verify_codes))
+    db.session.commit()
+    users = [x for x in service.users]
+    map(service.users.remove, users)
+    [service.users.remove(x) for x in users]
+    db.session.delete(service)
+    db.session.commit()
+    list(map(db.session.delete, users))
+    db.session.commit()

app/dao/templates_dao.py

@@ -1,7 +1,7 @@
 import uuid
 from app import db
 from app.models import (Template, Service)
-from sqlalchemy import asc
+from sqlalchemy import (asc, desc)
 
 from app.dao.dao_utils import (
     transactional,
@@ -22,11 +22,20 @@ def dao_update_template(template):
     db.session.add(template)
 
 
-def dao_get_template_by_id_and_service_id(template_id, service_id):
+def dao_get_template_by_id_and_service_id(template_id, service_id, version=None):
+    if version is not None:
+        return Template.get_history_model().query.filter_by(
+            id=template_id,
+            service_id=service_id,
+            version=version).one()
     return Template.query.filter_by(id=template_id, service_id=service_id).one()
 
 
-def dao_get_template_by_id(template_id):
+def dao_get_template_by_id(template_id, version=None):
+    if version is not None:
+        return Template.get_history_model().query.filter_by(
+            id=template_id,
+            version=version).one()
     return Template.query.filter_by(id=template_id).one()
 
 
@@ -36,3 +45,8 @@ def dao_get_all_templates_for_service(service_id):
     ).order_by(
         asc(Template.updated_at), asc(Template.created_at)
     ).all()
+
+
+def dao_get_template_versions(service_id, template_id):
+    history_model = Template.get_history_model()
+    return history_model.query.filter_by(service_id=service_id, id=template_id).order_by(desc(history_model.version))

app/dao/users_dao.py

@@ -67,6 +67,11 @@ def delete_model_user(user):
     db.session.commit()
 
 
+def delete_user_verify_codes(user):
+    VerifyCode.query.filter_by(user=user).delete()
+    db.session.commit()
+
+
 def get_model_users(user_id=None):
     if user_id:
         return User.query.filter_by(id=user_id).one()

app/notifications_statistics/rest.py

@@ -1,11 +1,18 @@
+from datetime import (date, timedelta)
 from flask import (
     Blueprint,
     jsonify,
     request
 )
 
-from app.dao.notifications_dao import dao_get_notification_statistics_for_service
-from app.schemas import notifications_statistics_schema
+from app.dao.notifications_dao import (
+    dao_get_notification_statistics_for_service,
+    dao_get_7_day_agg_notification_statistics_for_service
+)
+from app.schemas import (
+    notifications_statistics_schema,
+    week_aggregate_notification_statistics_schema
+)
 
 notifications_statistics = Blueprint(
     'notifications-statistics',
@@ -35,3 +42,32 @@ def get_all_notification_statistics_for_service(service_id):
 
     data, errors = notifications_statistics_schema.dump(statistics, many=True)
     return jsonify(data=data)
+
+
+@notifications_statistics.route('/seven_day_aggregate')
+def get_notification_statistics_for_service_seven_day_aggregate(service_id):
+    data, errors = week_aggregate_notification_statistics_schema.load(request.args)
+    if errors:
+        return jsonify(result='error', message=errors), 400
+    date_from = data['date_from'] if 'date_from' in data else date(date.today().year, 4, 1)
+    week_count = data['week_count'] if 'week_count' in data else 52
+    stats = dao_get_7_day_agg_notification_statistics_for_service(
+        service_id,
+        date_from,
+        week_count).all()
+    json_stats = []
+    for x in range(week_count - 1, -1, -1):
+        week_stats = stats.pop(0) if len(stats) > 0 and stats[0][0] == x else [x, 0, 0, 0, 0, 0, 0]
+        week_start = (date_from + timedelta(days=week_stats[0] * 7))
+        if week_start <= date.today():
+            json_stats.append({
+                'week_start': week_start.strftime('%Y-%m-%d'),
+                'week_end': (date_from + timedelta(days=(week_stats[0] * 7) + 6)).strftime('%Y-%m-%d'),
+                'emails_requested': week_stats[1],
+                'emails_delivered': week_stats[2],
+                'emails_failed': week_stats[3],
+                'sms_requested': week_stats[4],
+                'sms_delivered': week_stats[5],
+                'sms_failed': week_stats[6]
+            })
+    return jsonify(data=json_stats)

app/schemas.py

@@ -1,5 +1,6 @@
 from datetime import date
 from flask_marshmallow.fields import fields
+from sqlalchemy.orm import load_only
 
 from marshmallow import (
     post_load,
@@ -23,6 +24,20 @@ from app import models
 from app.dao.permissions_dao import permission_dao
 
 
+def _validate_positive_number(value, msg="Not a positive integer"):
+    try:
+        page_int = int(value)
+        if page_int < 1:
+            raise ValidationError(msg)
+    except:
+        raise ValidationError(msg)
+
+
+def _validate_not_in_future(dte, msg="Date cannot be in the future"):
+    if dte > date.today():
+        raise ValidationError(msg)
+
+
 # TODO I think marshmallow provides a better integration and error handling.
 # Would be better to replace functionality in dao with the marshmallow supported
 # functionality.
@@ -99,6 +114,11 @@ class BaseTemplateSchema(BaseSchema):
 class TemplateSchema(BaseTemplateSchema):
 
     created_by = field_for(models.Template, 'created_by', required=True)
+    versions = fields.Method("template_versions", dump_only=True)
+
+    def template_versions(self, template):
+        return [x.version for x in models.Template.get_history_model().query.filter_by(
+                id=template.id).options(load_only("version"))]
 
     @validates_schema
     def validate_type(self, data):
@@ -109,6 +129,11 @@ class TemplateSchema(BaseTemplateSchema):
                 raise ValidationError('Invalid template subject', 'subject')
 
 
+class TemplateHistorySchema(BaseTemplateSchema):
+
+    created_by = field_for(models.Template, 'created_by', required=True)
+
+
 class NotificationsStatisticsSchema(BaseSchema):
     class Meta:
         model = models.NotificationStatistics
@@ -260,21 +285,13 @@ class NotificationsFilterSchema(ma.Schema):
             in_data['status'] = [x.status for x in in_data['status']]
         return in_data
 
-    def _validate_positive_number(self, value):
-        try:
-            page_int = int(value)
-            if page_int < 1:
-                raise ValidationError("Not a positive integer")
-        except:
-            raise ValidationError("Not a positive integer")
-
     @validates('page')
     def validate_page(self, value):
-        self._validate_positive_number(value)
+        _validate_positive_number(value)
 
     @validates('page_size')
     def validate_page_size(self, value):
-        self._validate_positive_number(value)
+        _validate_positive_number(value)
 
 
 class TemplateStatisticsSchema(BaseSchema):
@@ -308,18 +325,6 @@ class ApiKeyHistorySchema(ma.Schema):
     created_by_id = fields.UUID()
 
 
-class TemplateHistorySchema(ma.Schema):
-    id = fields.UUID()
-    name = fields.String()
-    template_type = fields.String()
-    created_at = fields.DateTime()
-    updated_at = fields.DateTime()
-    content = fields.String()
-    service_id = fields.UUID()
-    subject = fields.String()
-    created_by_id = fields.UUID()
-
-
 class EventSchema(BaseSchema):
     class Meta:
         model = models.Event
@@ -330,17 +335,13 @@ class FromToDateSchema(ma.Schema):
     date_from = fields.Date()
     date_to = fields.Date()
 
-    def _validate_not_in_future(self, dte):
-        if dte > date.today():
-            raise ValidationError('Date cannot be in the future')
-
     @validates('date_from')
     def validate_date_from(self, value):
-        self._validate_not_in_future(value)
+        _validate_not_in_future(value)
 
     @validates('date_to')
     def validate_date_to(self, value):
-        self._validate_not_in_future(value)
+        _validate_not_in_future(value)
 
     @validates_schema
     def validate_dates(self, data):
@@ -350,6 +351,20 @@ class FromToDateSchema(ma.Schema):
             raise ValidationError("date_from needs to be greater than date_to")
 
 
+class WeekAggregateNotificationStatisticsSchema(ma.Schema):
+
+    date_from = fields.Date()
+    week_count = fields.Int()
+
+    @validates('date_from')
+    def validate_date_from(self, value):
+        _validate_not_in_future(value)
+
+    @validates('week_count')
+    def validate_week_count(self, value):
+        _validate_positive_number(value)
+
+
 user_schema = UserSchema()
 user_schema_load_json = UserSchema(load_json=True)
 service_schema = ServiceSchema()
@@ -380,3 +395,4 @@ template_history_schema = TemplateHistorySchema()
 event_schema = EventSchema()
 from_to_date_schema = FromToDateSchema()
 provider_details_schema = ProviderDetailsSchema()
+week_aggregate_notification_statistics_schema = WeekAggregateNotificationStatisticsSchema()

app/template/rest.py

@@ -11,11 +11,12 @@ from app.dao.templates_dao import (
     dao_update_template,
     dao_create_template,
     dao_get_template_by_id_and_service_id,
-    dao_get_all_templates_for_service
+    dao_get_all_templates_for_service,
+    dao_get_template_versions
 )
 from notifications_utils.template import Template
 from app.dao.services_dao import dao_fetch_service_by_id
-from app.schemas import template_schema
+from app.schemas import (template_schema, template_history_schema)
 
 template = Blueprint('template', __name__, url_prefix='/service/<uuid:service_id>/template')
 
@@ -96,5 +97,27 @@ def get_template_by_id_and_service_id(service_id, template_id):
     return jsonify(data=data)
 
 
+@template.route('/<uuid:template_id>/version/<int:version>')
+def get_template_version(service_id, template_id, version):
+    fetched_template = dao_get_template_by_id_and_service_id(
+        template_id=template_id,
+        service_id=service_id,
+        version=version
+    )
+    data, errors = template_history_schema.dump(fetched_template)
+    if errors:
+        return json_resp(result='error', message=errors), 400
+    return jsonify(data=data)
+
+
+@template.route('/<uuid:template_id>/version')
+def get_template_versions(service_id, template_id):
+    fetched_templates = dao_get_template_versions(service_id, template_id)
+    data, errors = template_history_schema.dump(fetched_templates, many=True)
+    if errors:
+        return json_resp(result='error', message=errors), 400
+    return jsonify(data=data)
+
+
 def _strip_html(content):
     return bleach.clean(content, tags=[], strip=True)