[WIP] On create of notification. Upsert record for template stats

recording usages of template by day.

app/celery/tasks.py

@@ -1,9 +1,34 @@
-from app import create_uuid, DATETIME_FORMAT, DATE_FORMAT
-from app import notify_celery, encryption, firetext_client, aws_ses_client
+from datetime import datetime
+
+from flask import current_app
+from sqlalchemy.exc import SQLAlchemyError
+
 from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
 from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
+
+from utils.template import Template
+
+from utils.recipients import (
+    RecipientCSV,
+    validate_and_format_phone_number
+)
+
+from app import (
+    create_uuid,
+    DATETIME_FORMAT,
+    DATE_FORMAT,
+    notify_celery,
+    encryption,
+    firetext_client,
+    aws_ses_client
+)
+
+from app.aws import s3
+from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
+
 from app.dao.notifications_dao import (
     dao_create_notification,
     dao_update_notification,
@@ -12,21 +37,22 @@ from app.dao.notifications_dao import (
     dao_get_notification_statistics_for_service_and_day,
     update_notification_reference_by_id
 )
-from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
-from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
+
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id
+)
+
 from app.models import (
     Notification,
     TEMPLATE_TYPE_EMAIL,
     TEMPLATE_TYPE_SMS
 )
-from flask import current_app
-from sqlalchemy.exc import SQLAlchemyError
-from app.aws import s3
-from datetime import datetime
-from utils.template import Template
-from utils.recipients import RecipientCSV, validate_and_format_phone_number
-from app.validation import (allowed_send_to_email, allowed_send_to_number)
+
+from app.validation import (
+    allowed_send_to_email,
+    allowed_send_to_number
+)
 
 
 @notify_celery.task(name="delete-verify-codes")

app/dao/notifications_dao.py

@@ -1,15 +1,28 @@
+from sqlalchemy import desc
+from datetime import (
+    datetime,
+    timedelta,
+    date
+)
+
 from flask import current_app
+
 from app import db
 from app.models import (
     Notification,
     Job,
     NotificationStatistics,
+    TemplateStatistics,
     TEMPLATE_TYPE_SMS,
     TEMPLATE_TYPE_EMAIL,
-    Template)
-from sqlalchemy import desc
-from datetime import datetime, timedelta
-from app.clients import (STATISTICS_FAILURE, STATISTICS_DELIVERED, STATISTICS_REQUESTED)
+    Template
+)
+
+from app.clients import (
+    STATISTICS_FAILURE,
+    STATISTICS_DELIVERED,
+    STATISTICS_REQUESTED
+)
 
 
 def dao_get_notification_statistics_for_service(service_id):
@@ -48,6 +61,18 @@ def dao_create_notification(notification, notification_type):
                 emails_requested=1 if notification_type == TEMPLATE_TYPE_EMAIL else 0
             )
             db.session.add(stats)
+
+        update_count = db.session.query(TemplateStatistics).filter_by(
+            day=date.today(),
+            service_id=notification.service_id,
+            template_id=notification.template_id
+        ).update({'usage_count': TemplateStatistics.usage_count + 1})
+
+        if update_count == 0:
+            template_stats = TemplateStatistics(template_id=notification.template_id,
+                                                service_id=notification.service_id)
+            db.session.add(template_stats)
+
         db.session.add(notification)
         db.session.commit()
     except:

app/models.py

@@ -1,9 +1,14 @@
 import uuid
+import datetime
 
 from sqlalchemy import UniqueConstraint, Sequence
+
 from . import db
-import datetime
-from sqlalchemy.dialects.postgresql import (UUID, ARRAY)
+from sqlalchemy.dialects.postgresql import (
+    UUID,
+    ARRAY
+)
+
 from app.encryption import (
     hashpw,
     check_hash
@@ -346,3 +351,14 @@ class Permission(db.Model):
     __table_args__ = (
         UniqueConstraint('service_id', 'user_id', 'permission', name='uix_service_user_permission'),
     )
+
+
+class TemplateStatistics(db.Model):
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
+    service = db.relationship('Service', backref=db.backref('template_statics', lazy='dynamic'))
+    template_id = db.Column(db.BigInteger, db.ForeignKey('templates.id'), index=True, nullable=False, unique=False)
+    template = db.relationship('Template')
+    usage_count = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=1)
+    day = db.Column(db.Date, index=True, nullable=False, unique=False, default=datetime.date.today)

app/notifications/rest.py

@@ -20,11 +20,11 @@ from app.dao import (
     services_dao,
     notifications_dao
 )
+
 from app.schemas import (
     email_notification_schema,
     sms_template_notification_schema,
     notification_status_schema,
-    template_schema,
     notifications_filter_schema
 )
 from app.celery.tasks import send_sms, send_email

migrations/versions/0044_add_template_stats.py

@@ -0,0 +1,42 @@
+"""empty message
+
+Revision ID: 0044_add_template_stats
+Revises: 0043_add_view_activity
+Create Date: 2016-03-31 12:05:19.630792
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0044_add_template_stats'
+down_revision = '0043_add_view_activity'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('template_statistics',
+    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('template_id', sa.BigInteger(), nullable=False),
+    sa.Column('usage_count', sa.BigInteger(), nullable=False),
+    sa.Column('day', sa.Date(), nullable=False),
+    sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+    sa.ForeignKeyConstraint(['template_id'], ['templates.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_template_statistics_day'), 'template_statistics', ['day'], unique=False)
+    op.create_index(op.f('ix_template_statistics_service_id'), 'template_statistics', ['service_id'], unique=False)
+    op.create_index(op.f('ix_template_statistics_template_id'), 'template_statistics', ['template_id'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_template_statistics_template_id'), table_name='template_statistics')
+    op.drop_index(op.f('ix_template_statistics_service_id'), table_name='template_statistics')
+    op.drop_index(op.f('ix_template_statistics_day'), table_name='template_statistics')
+    op.drop_table('template_statistics')
+    ### end Alembic commands ###

tests/app/dao/test_notification_dao.py

@@ -1,12 +1,19 @@
-import pytest
+from datetime import datetime, timedelta, date
 import uuid
-from app import (
-    DATE_FORMAT
-)
+
+import pytest
+
+from app import DATE_FORMAT
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError, IntegrityError
-from app.models import Notification, Job, NotificationStatistics
-from datetime import datetime, timedelta
+
+from app.models import (
+    Notification,
+    Job,
+    NotificationStatistics,
+    TemplateStatistics
+)
+
 from app.dao.notifications_dao import (
     dao_create_notification,
     dao_update_notification,
@@ -21,6 +28,7 @@ from app.dao.notifications_dao import (
     update_notification_reference_by_id,
     update_notification_status_by_reference
 )
+
 from tests.app.conftest import sample_job
 from tests.app.conftest import sample_notification
 
@@ -38,6 +46,7 @@ def test_should_by_able_to_update_status_by_reference(sample_email_template):
         'service': sample_email_template.service,
         'service_id': sample_email_template.service.id,
         'template': sample_email_template,
+        'template_id': sample_email_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -97,6 +106,7 @@ def test_should_be_able_to_record_statistics_failure_for_email(sample_email_temp
         'service': sample_email_template.service,
         'service_id': sample_email_template.service.id,
         'template': sample_email_template,
+        'template_id': sample_email_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -132,6 +142,7 @@ def test_should_be_able_to_get_statistics_for_a_service(sample_template):
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -158,6 +169,7 @@ def test_should_be_able_to_get_statistics_for_a_service_for_a_day(sample_templat
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': now
     }
 
@@ -183,6 +195,7 @@ def test_should_return_none_if_no_statistics_for_a_service_for_a_day(sample_temp
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': now
     }
 
@@ -199,6 +212,7 @@ def test_should_be_able_to_get_all_statistics_for_a_service(sample_template):
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -220,7 +234,8 @@ def test_should_be_able_to_get_all_statistics_for_a_service_for_several_days(sam
         'to': '+44709123456',
         'service': sample_template.service,
         'service_id': sample_template.service.id,
-        'template': sample_template
+        'template': sample_template,
+        'template_id': sample_template.id,
     }
 
     today = datetime.utcnow()
@@ -259,14 +274,18 @@ def test_should_be_empty_list_if_no_statistics_for_a_service(sample_service):
     assert len(dao_get_notification_statistics_for_service(sample_service.id)) == 0
 
 
-def test_save_notification_and_create_sms_stats(sample_template, sample_job):
+def test_save_notification_creates_sms_and_template_stats(sample_template, sample_job):
     assert Notification.query.count() == 0
+    assert NotificationStatistics.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
+
     data = {
         'to': '+44709123456',
         'job_id': sample_job.id,
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -291,15 +310,27 @@ def test_save_notification_and_create_sms_stats(sample_template, sample_job):
     assert stats.emails_requested == 0
     assert stats.sms_requested == 1
 
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_template.service.id,
+                                                     TemplateStatistics.template_id == sample_template.id).first()
+
+    assert template_stats.service_id == sample_template.service.id
+    assert template_stats.template_id == sample_template.id
+    assert template_stats.usage_count == 1
+
+
+def test_save_notification_and_create_email_and_template_stats(sample_email_template, sample_job):
 
-def test_save_notification_and_create_email_stats(sample_email_template, sample_job):
     assert Notification.query.count() == 0
+    assert NotificationStatistics.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
+
     data = {
         'to': '+44709123456',
         'job_id': sample_job.id,
         'service': sample_email_template.service,
         'service_id': sample_email_template.service.id,
         'template': sample_email_template,
+        'template_id': sample_email_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -324,6 +355,13 @@ def test_save_notification_and_create_email_stats(sample_email_template, sample_
     assert stats.emails_requested == 1
     assert stats.sms_requested == 0
 
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_email_template.service.id,
+                                                     TemplateStatistics.template_id == sample_email_template.id).first()  # noqa
+
+    assert template_stats.service_id == sample_email_template.service.id
+    assert template_stats.template_id == sample_email_template.id
+    assert template_stats.usage_count == 1
+
 
 @freeze_time("2016-01-01 00:00:00.000000")
 def test_save_notification_handles_midnight_properly(sample_template, sample_job):
@@ -334,6 +372,7 @@ def test_save_notification_handles_midnight_properly(sample_template, sample_job
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -358,6 +397,7 @@ def test_save_notification_handles_just_before_midnight_properly(sample_template
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -381,6 +421,7 @@ def test_save_notification_and_increment_email_stats(sample_email_template, samp
         'service': sample_email_template.service,
         'service_id': sample_email_template.service.id,
         'template': sample_email_template,
+        'template_id': sample_email_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -417,6 +458,7 @@ def test_save_notification_and_increment_sms_stats(sample_template, sample_job):
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -455,6 +497,7 @@ def test_not_save_notification_and_not_create_stats_on_commit_error(sample_templ
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -465,6 +508,7 @@ def test_not_save_notification_and_not_create_stats_on_commit_error(sample_templ
     assert Notification.query.count() == 0
     assert Job.query.get(sample_job.id).notifications_sent == 0
     assert NotificationStatistics.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
 
 
 def test_save_notification_and_increment_job(sample_template, sample_job):
@@ -475,6 +519,7 @@ def test_save_notification_and_increment_job(sample_template, sample_job):
         'service': sample_template.service,
         'service_id': sample_template.service.id,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -509,6 +554,7 @@ def test_should_not_increment_job_if_notification_fails_to_persist(sample_templa
         'service_id': sample_template.service.id,
         'service': sample_template.service,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -539,6 +585,7 @@ def test_save_notification_and_increment_correct_job(notify_db, notify_db_sessio
         'service_id': sample_template.service.id,
         'service': sample_template.service,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -565,6 +612,7 @@ def test_save_notification_with_no_job(sample_template):
         'service_id': sample_template.service.id,
         'service': sample_template.service,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -596,6 +644,7 @@ def test_save_notification_no_job_id(sample_template):
         'service_id': sample_template.service.id,
         'service': sample_template.service,
         'template': sample_template,
+        'template_id': sample_template.id,
         'created_at': datetime.utcnow()
     }
 
@@ -688,3 +737,106 @@ def test_should_not_delete_failed_notifications_before_seven_days(notify_db, not
     delete_failed_notifications_created_more_than_a_week_ago()
     assert len(Notification.query.all()) == 1
     assert Notification.query.first().to == 'valid'
+
+
+@freeze_time("2016-03-30")
+def test_save_new_notification_creates_template_stats(sample_template, sample_job):
+    assert Notification.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
+    data = {
+        'to': '+44709123456',
+        'job_id': sample_job.id,
+        'service': sample_template.service,
+        'service_id': sample_template.service.id,
+        'template': sample_template,
+        'template_id': sample_template.id,
+        'created_at': datetime.utcnow()
+    }
+
+    notification = Notification(**data)
+    dao_create_notification(notification, sample_template.template_type)
+
+    assert TemplateStatistics.query.count() == 1
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_template.service.id,
+                                                     TemplateStatistics.template_id == sample_template.id).first()
+    assert template_stats.service_id == sample_template.service.id
+    assert template_stats.template_id == sample_template.id
+    assert template_stats.usage_count == 1
+    assert template_stats.day == date(2016, 3, 30)
+
+
+@freeze_time("2016-03-30")
+def test_save_new_notification_creates_template_stats_per_day(sample_template, sample_job):
+    assert Notification.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
+    data = {
+        'to': '+44709123456',
+        'job_id': sample_job.id,
+        'service': sample_template.service,
+        'service_id': sample_template.service.id,
+        'template': sample_template,
+        'template_id': sample_template.id,
+        'created_at': datetime.utcnow()
+    }
+
+    notification = Notification(**data)
+    dao_create_notification(notification, sample_template.template_type)
+
+    assert TemplateStatistics.query.count() == 1
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_template.service.id,
+                                                     TemplateStatistics.template_id == sample_template.id).first()
+    assert template_stats.service_id == sample_template.service.id
+    assert template_stats.template_id == sample_template.id
+    assert template_stats.usage_count == 1
+    assert template_stats.day == date(2016, 3, 30)
+
+    # move on one day
+    with freeze_time('2016-03-31'):
+        assert TemplateStatistics.query.count() == 1
+        new_notification = Notification(**data)
+        dao_create_notification(new_notification, sample_template.template_type)
+
+    assert TemplateStatistics.query.count() == 2
+    first_stats = TemplateStatistics.query.filter(TemplateStatistics.day == datetime(2016, 3, 30)).first()
+    second_stats = TemplateStatistics.query.filter(TemplateStatistics.day == datetime(2016, 3, 31)).first()
+
+    assert first_stats.template_id == second_stats.template_id
+    assert first_stats.service_id == second_stats.service_id
+
+    assert first_stats.day == date(2016, 3, 30)
+    assert first_stats.usage_count == 1
+
+    assert second_stats.day == date(2016, 3, 31)
+    assert second_stats.usage_count == 1
+
+
+def test_save_another_notification_increments_template_stats(sample_template, sample_job):
+    assert Notification.query.count() == 0
+    assert TemplateStatistics.query.count() == 0
+    data = {
+        'to': '+44709123456',
+        'job_id': sample_job.id,
+        'service': sample_template.service,
+        'service_id': sample_template.service.id,
+        'template': sample_template,
+        'template_id': sample_template.id,
+        'created_at': datetime.utcnow()
+    }
+
+    notification_1 = Notification(**data)
+    notification_2 = Notification(**data)
+    dao_create_notification(notification_1, sample_template.template_type)
+
+    assert TemplateStatistics.query.count() == 1
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_template.service.id,
+                                                     TemplateStatistics.template_id == sample_template.id).first()
+    assert template_stats.service_id == sample_template.service.id
+    assert template_stats.template_id == sample_template.id
+    assert template_stats.usage_count == 1
+
+    dao_create_notification(notification_2, sample_template.template_type)
+
+    assert TemplateStatistics.query.count() == 1
+    template_stats = TemplateStatistics.query.filter(TemplateStatistics.service_id == sample_template.service.id,
+                                                     TemplateStatistics.template_id == sample_template.id).first()
+    assert template_stats.usage_count == 2