Merge branch 'master' into reset-password

Conflicts:
	app/schemas.py
	tests/app/celery/test_tasks.py

app/__init__.py

@@ -13,6 +13,8 @@ from app.clients.sms.firetext import FiretextClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.encryption import Encryption
 
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
+
 db = SQLAlchemy()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
@@ -49,6 +51,7 @@ def create_app():
     from app.invite.rest import invite as invite_blueprint
     from app.permission.rest import permission as permission_blueprint
     from app.accept_invite.rest import accept_invite
+    from app.notifications_statistics.rest import notifications_statistics as notifications_statistics_blueprint
 
     application.register_blueprint(service_blueprint, url_prefix='/service')
     application.register_blueprint(user_blueprint, url_prefix='/user')
@@ -59,6 +62,7 @@ def create_app():
     application.register_blueprint(invite_blueprint)
     application.register_blueprint(permission_blueprint, url_prefix='/permission')
     application.register_blueprint(accept_invite, url_prefix='/invite')
+    application.register_blueprint(notifications_statistics_blueprint)
 
     return application
 

app/authentication/auth.py

@@ -61,11 +61,6 @@ def fetch_client(client):
             "client": client,
             "secret": [current_app.config.get('ADMIN_CLIENT_SECRET')]
         }
-    elif client == current_app.config.get('DELIVERY_CLIENT_USER_NAME'):
-        return {
-            "client": client,
-            "secret": [current_app.config.get('DELIVERY_CLIENT_SECRET')]
-        }
     else:
         return {
             "client": client,

app/celery/tasks.py

@@ -1,4 +1,4 @@
-from app import create_uuid
+from app import create_uuid, DATETIME_FORMAT
 from app import notify_celery, encryption, firetext_client, aws_ses_client
 from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
@@ -6,7 +6,7 @@ from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.notifications_dao import dao_create_notification, dao_update_notification
 from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
-from app.models import Notification
+from app.models import Notification, TEMPLATE_TYPE_EMAIL, TEMPLATE_TYPE_SMS
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
@@ -39,7 +39,7 @@ def process_job(job_id):
                 str(job.service_id),
                 str(create_uuid()),
                 encrypted,
-                str(datetime.utcnow())),
+                datetime.utcnow().strftime(DATETIME_FORMAT)),
                 queue='bulk-sms'
             )
 
@@ -50,7 +50,7 @@ def process_job(job_id):
                 job.template.subject,
                 "{}@{}".format(job.service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
                 encrypted,
-                str(datetime.utcnow())),
+                datetime.utcnow().strftime(DATETIME_FORMAT)),
                 queue='bulk-email')
 
     finished = datetime.utcnow()
@@ -89,12 +89,12 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
             service_id=service_id,
             job_id=notification.get('job', None),
             status=status,
-            created_at=created_at,
+            created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
             sent_by=client.get_name()
         )
 
-        dao_create_notification(notification_db_object)
+        dao_create_notification(notification_db_object, TEMPLATE_TYPE_SMS)
 
         if can_send:
             try:
@@ -159,11 +159,11 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
             service_id=service_id,
             job_id=notification.get('job', None),
             status=status,
-            created_at=created_at,
+            created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
             sent_by=client.get_name()
         )
-        dao_create_notification(notification_db_object)
+        dao_create_notification(notification_db_object, TEMPLATE_TYPE_EMAIL)
 
         if can_send:
             try:

app/dao/notifications_dao.py

@@ -1,16 +1,59 @@
 from flask import current_app
 from app import db
-from app.models import Notification, Job
+from app.models import Notification, Job, NotificationStatistics, TEMPLATE_TYPE_SMS, TEMPLATE_TYPE_EMAIL
 from sqlalchemy import desc
+from datetime import datetime
 
 
-def dao_create_notification(notification):
-    if notification.job_id:
-        db.session.query(Job).filter_by(
-            id=notification.job_id
-        ).update({Job.notifications_sent: Job.notifications_sent + 1})
-    db.session.add(notification)
-    db.session.commit()
+def dao_get_notification_statistics_for_service(service_id):
+    return NotificationStatistics.query.filter_by(
+        service_id=service_id
+    ).order_by(desc(NotificationStatistics.day)).all()
+
+
+def dao_create_notification(notification, notification_type):
+    try:
+        if notification.job_id:
+            update_job_sent_count(notification)
+
+        if update_notification_stats(notification, notification_type) == 0:
+            stats = NotificationStatistics(
+                day=notification.created_at.strftime('%Y-%m-%d'),
+                service_id=notification.service_id,
+                sms_requested=1 if notification_type == TEMPLATE_TYPE_SMS else 0,
+                emails_requested=1 if notification_type == TEMPLATE_TYPE_EMAIL else 0
+            )
+            db.session.add(stats)
+        db.session.add(notification)
+        db.session.commit()
+    except:
+        db.session.rollback()
+        raise
+
+
+def update_notification_stats(notification, notification_type):
+    if notification_type == TEMPLATE_TYPE_SMS:
+        update = {
+            NotificationStatistics.sms_requested: NotificationStatistics.sms_requested + 1
+        }
+    else:
+        update = {
+            NotificationStatistics.emails_requested: NotificationStatistics.emails_requested + 1
+        }
+
+    return db.session.query(NotificationStatistics).filter_by(
+        day=notification.created_at.strftime('%Y-%m-%d'),
+        service_id=notification.service_id
+    ).update(update)
+
+
+def update_job_sent_count(notification):
+    db.session.query(Job).filter_by(
+        id=notification.job_id
+    ).update({
+        Job.notifications_sent: Job.notifications_sent + 1,
+        Job.updated_at: datetime.utcnow()
+    })
 
 
 def dao_update_notification(notification):

app/models.py

@@ -107,7 +107,30 @@ class ApiKey(db.Model):
     )
 
 
-TEMPLATE_TYPES = ['sms', 'email', 'letter']
+class NotificationStatistics(db.Model):
+    __tablename__ = 'notification_statistics'
+
+    id = db.Column(db.Integer, primary_key=True)
+    day = db.Column(db.String(255), nullable=False)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref=db.backref('service_notification_stats', lazy='dynamic'))
+    emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+
+    __table_args__ = (
+        UniqueConstraint('service_id', 'day', name='uix_service_to_day'),
+    )
+
+
+TEMPLATE_TYPE_SMS = 'sms'
+TEMPLATE_TYPE_EMAIL = 'email'
+TEMPLATE_TYPE_LETTER = 'letter'
+
+TEMPLATE_TYPES = [TEMPLATE_TYPE_SMS, TEMPLATE_TYPE_EMAIL, TEMPLATE_TYPE_LETTER]
 
 
 class Template(db.Model):

app/notifications/rest.py

@@ -10,7 +10,7 @@ from flask import (
 
 from utils.template import Template, NeededByTemplateError, NoPlaceholderForDataError
 
-from app import api_user, encryption, create_uuid
+from app import api_user, encryption, create_uuid, DATETIME_FORMAT
 from app.authentication.auth import require_admin
 from app.dao import (
     templates_dao,
@@ -178,7 +178,7 @@ def send_notification(notification_type):
             service_id,
             notification_id,
             encryption.encrypt(notification),
-            str(datetime.utcnow())
+            datetime.utcnow().strftime(DATETIME_FORMAT)
         ), queue='sms')
     else:
         if service.restricted and notification['to'] not in [user.email_address for user in service.users]:
@@ -190,6 +190,6 @@ def send_notification(notification_type):
             template.subject,
             "{}@{}".format(service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
             encryption.encrypt(notification),
-            str(datetime.utcnow())
+            datetime.utcnow().strftime(DATETIME_FORMAT)
         ), queue='email')
     return jsonify({'notification_id': notification_id}), 201

app/notifications_statistics/rest.py

@@ -0,0 +1,25 @@
+from flask import (
+    Blueprint,
+    jsonify,
+)
+
+from app.dao.notifications_dao import (
+    dao_get_notification_statistics_for_service
+)
+from app.schemas import notifications_statistics_schema
+
+notifications_statistics = Blueprint(
+    'notifications-statistics',
+    __name__, url_prefix='/service/<service_id>/notifications-statistics'
+)
+
+from app.errors import register_errors
+
+register_errors(notifications_statistics)
+
+
+@notifications_statistics.route('', methods=['GET'])
+def get_all_templates_for_service(service_id):
+    templates = dao_get_notification_statistics_for_service(service_id=service_id)
+    data, errors = notifications_statistics_schema.dump(templates, many=True)
+    return jsonify(data=data)

app/schemas.py

@@ -33,19 +33,6 @@ class BaseSchema(ma.ModelSchema):
         assert key is not None, "Envelope key undefined"
         return key
 
-    # Code to envelope the input and response.
-    # TOBE added soon.
-
-    # @pre_load(pass_many=True)
-    # def unwrap_envelope(self, data, many):
-    #     key = self.get_envelope_key(many)
-    #     return data[key]
-
-    # @post_dump(pass_many=True)
-    # def wrap_with_envelope(self, data, many):
-    #     key = self.get_envelope_key(many)
-    #     return {key: data}
-
     @post_load
     def make_instance(self, data):
         """Deserialize data to an instance of the model. Update an existing row
@@ -92,6 +79,11 @@ class TemplateSchema(BaseSchema):
         exclude = ("updated_at", "created_at", "service_id", "jobs")
 
 
+class NotificationsStatisticsSchema(BaseSchema):
+    class Meta:
+        model = models.NotificationStatistics
+
+
 class ApiKeySchema(BaseSchema):
     class Meta:
         model = models.ApiKey
@@ -228,3 +220,4 @@ notification_status_schema_load_json = NotificationStatusSchema(load_json=True)
 invited_user_schema = InvitedUserSchema()
 permission_schema = PermissionSchema()
 email_data_request_schema = EmailDataSchema()
+notifications_statistics_schema = NotificationsStatisticsSchema()