Merge branch 'master' into new-staging

app/dao/services_dao.py

@@ -1,4 +1,5 @@
 import uuid
+from datetime import date
 
 from sqlalchemy import asc, func
 from sqlalchemy.orm import joinedload
@@ -136,6 +137,16 @@ def delete_service_and_all_associated_db_objects(service):
 
 
 def dao_fetch_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).all()
+
+
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def _stats_for_service_query(service_id):
     return db.session.query(
         Notification.notification_type,
         Notification.status,
@@ -145,4 +156,4 @@ def dao_fetch_stats_for_service(service_id):
     ).group_by(
         Notification.notification_type,
         Notification.status,
-    ).all()
+    )

app/service/rest.py

@@ -22,7 +22,8 @@ from app.dao.services_dao import (
     dao_fetch_all_services_by_user,
     dao_add_user_to_service,
     dao_remove_user_from_service,
-    dao_fetch_stats_for_service
+    dao_fetch_stats_for_service,
+    dao_fetch_todays_stats_for_service
 )
 from app.dao import notifications_dao
 from app.dao.provider_statistics_dao import get_fragment_count
@@ -60,8 +61,8 @@ def get_services():
 
 @service.route('/<uuid:service_id>', methods=['GET'])
 def get_service_by_id(service_id):
-    if 'detailed' in request.args:
+    if request.args.get('detailed') == 'True':
-        return get_detailed_service(service_id)
+        return get_detailed_service(service_id, today_only=request.args.get('today_only') == 'True')
     else:
         fetched = dao_fetch_service_by_id(service_id)
 
@@ -235,9 +236,10 @@ def get_all_notifications_for_service(service_id):
     ), 200
 
 
-def get_detailed_service(service_id):
+def get_detailed_service(service_id, today_only=False):
     service = dao_fetch_service_by_id(service_id)
-    statistics = dao_fetch_stats_for_service(service_id)
+    stats_fn = dao_fetch_todays_stats_for_service if today_only else dao_fetch_stats_for_service
+    statistics = stats_fn(service_id)
     service.statistics = format_statistics(statistics)
     data = detailed_service_schema.dump(service).data
     return jsonify(data=data)

migrations/versions/0043_notification_indexes.py

@@ -0,0 +1,44 @@
+"""empty message
+
+Revision ID: 0043_notification_indexes
+Revises: 0042_notification_history
+Create Date: 2016-08-01 10:37:41.198070
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0043_notification_indexes'
+down_revision = '0042_notification_history'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_index(op.f('ix_notifications_created_at'), 'notifications', ['created_at'])
+    op.create_index(op.f('ix_notification_history_created_at'), 'notification_history', ['created_at'])
+
+    op.create_index(op.f('ix_notifications_status'), 'notifications', ['status'])
+    op.create_index(op.f('ix_notification_history_status'), 'notification_history', ['status'])
+
+    op.create_index(op.f('ix_notifications_notification_type'), 'notifications', ['notification_type'])
+    op.create_index(op.f('ix_notification_history_notification_type'), 'notification_history', ['notification_type'])
+
+    op.create_index(
+        'ix_notification_history_week_created',
+        'notification_history',
+        [sa.text("date_trunc('week', created_at)")]
+    )
+
+
+def downgrade():
+    op.drop_index(op.f('ix_notifications_created_at'), table_name='notifications')
+    op.drop_index(op.f('ix_notification_history_created_at'), table_name='notification_history')
+
+    op.drop_index(op.f('ix_notifications_status'), table_name='notifications')
+    op.drop_index(op.f('ix_notification_history_status'), table_name='notification_history')
+
+    op.drop_index(op.f('ix_notifications_notification_type'), table_name='notifications')
+    op.drop_index(op.f('ix_notification_history_notification_type'), table_name='notification_history')
+
+    op.drop_index(op.f('ix_notification_history_week_created'), table_name='notification_history')

migrations/versions/0044_jobs_to_notification_hist.py

@@ -0,0 +1,78 @@
+"""empty message
+
+Revision ID: 0044_jobs_to_notification_hist
+Revises: 0043_notification_indexes
+Create Date: 2016-07-15 13:28:41.441009
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0044_jobs_to_notification_hist'
+down_revision = '0043_notification_indexes'
+
+from alembic import op
+
+from sqlalchemy.orm.session import Session
+
+import uuid
+import datetime
+from app.models import Job, Template, NotificationHistory
+
+
+def upgrade():
+    session = Session(bind=op.get_bind())
+
+    go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
+    notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
+    jobs = session.query(Job).join(Template).filter(Job.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
+                                                    Job.created_at >= go_live,
+                                                    Job.created_at < notifications_history_start_date).all()
+
+    for job in jobs:
+        for i in range(0, job.notifications_delivered):
+            notification = NotificationHistory(id=uuid.uuid4(),
+                                               job_id=job.id,
+                                               service_id=job.service_id,
+                                               template_id=job.template.id,
+                                               template_version=job.template_version,
+                                               key_type='normal',
+                                               content_char_count=len(job.template.content),
+                                               notification_type=job.template.template_type,
+                                               created_at=job.created_at,
+                                               sent_at=job.processing_finished,
+                                               sent_by='ses' if job.template.template_type == 'email' else 'mmg',
+                                               status='delivered')
+
+            session.add(notification)
+
+        for i in range(0, job.notifications_failed):
+            notification = NotificationHistory(id=uuid.uuid4(),
+                                               job_id=job.id,
+                                               service_id=job.service_id,
+                                               template_id=job.template.id,
+                                               template_version=job.template_version,
+                                               key_type='normal',
+                                               content_char_count=len(job.template.content),
+                                               notification_type=job.template.template_type,
+                                               created_at=job.created_at,
+                                               sent_at=job.processing_finished,
+                                               sent_by='ses' if job.template.template_type == 'email' else 'mmg',
+                                               status='permanent-failure')
+            session.add(notification)
+        session.commit()
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    session = Session(bind=op.get_bind())
+
+    go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
+    notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
+
+    session.query(NotificationHistory).filter(
+        NotificationHistory.created_at >= go_live,
+        NotificationHistory.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
+        NotificationHistory.created_at < notifications_history_start_date).delete()
+
+    session.commit()
+    ### end Alembic commands ###

requirements.txt

@@ -23,4 +23,4 @@ statsd==3.2.1
 git+https://github.com/alphagov/notifications-python-client.git@1.0.0#egg=notifications-python-client==1.0.0
 
 
-git+https://github.com/alphagov/notifications-utils.git@8.6.0#egg=notifications-utils==8.6.0
+git+https://github.com/alphagov/notifications-utils.git@8.7.2#egg=notifications-utils==8.7.2

tests/app/conftest.py

@@ -319,12 +319,14 @@ def sample_notification(notify_db,
                         to_field=None,
                         status='created',
                         reference=None,
-                        created_at=datetime.utcnow(),
+                        created_at=None,
                         content_char_count=160,
                         create=True,
                         personalisation=None,
                         api_key_id=None,
                         key_type=KEY_TYPE_NORMAL):
+    if created_at is None:
+        created_at = datetime.utcnow()
     if service is None:
         service = sample_service(notify_db, notify_db_session)
     if template is None:

tests/app/dao/test_services_dao.py

@@ -3,6 +3,7 @@ import pytest
 
 from sqlalchemy.orm.exc import FlushError, NoResultFound
 from sqlalchemy.exc import IntegrityError
+from freezegun import freeze_time
 
 from app import db
 from app.dao.services_dao import (
@@ -14,7 +15,8 @@ from app.dao.services_dao import (
     dao_fetch_all_services_by_user,
     dao_update_service,
     delete_service_and_all_associated_db_objects,
-    dao_fetch_stats_for_service
+    dao_fetch_stats_for_service,
+    dao_fetch_todays_stats_for_service
 )
 from app.dao.users_dao import save_model_user
 from app.models import (
@@ -420,3 +422,22 @@ def test_fetch_stats_counts_correctly(notify_db, notify_db_session, sample_templ
     assert stats[2].notification_type == 'sms'
     assert stats[2].status == 'created'
     assert stats[2].count == 1
+
+
+def test_fetch_stats_for_today_only_includes_today(notify_db, notify_db_session, sample_template):
+    # two created email, one failed email, and one created sms
+    with freeze_time('2001-01-01T23:59:00'):
+        just_before_midnight_yesterday = create_notification(notify_db, None, to_field='1', status='delivered')
+
+    with freeze_time('2001-01-02T00:01:00'):
+        just_after_midnight_today = create_notification(notify_db, None, to_field='2', status='failed')
+
+    with freeze_time('2001-01-02T12:00:00'):
+        right_now = create_notification(notify_db, None, to_field='3', status='created')
+
+        stats = dao_fetch_todays_stats_for_service(sample_template.service.id)
+
+    stats = {row.status: row.count for row in stats}
+    assert 'delivered' not in stats
+    assert stats['failed'] == 1
+    assert stats['created'] == 1

tests/app/service/test_rest.py

@@ -4,6 +4,7 @@ import uuid
 
 import pytest
 from flask import url_for
+from freezegun import freeze_time
 
 from app.dao.users_dao import save_model_user
 from app.dao.services_dao import dao_remove_user_from_service
@@ -1094,23 +1095,36 @@ def test_set_sms_sender_for_service_rejects_invalid_characters(notify_api, sampl
             assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']}
 
 
-def test_get_detailed_service(notify_api, sample_service, sample_notification):
+@pytest.mark.parametrize('today_only,stats', [
+    ('False', {
+        'requested': 2,
+        'delivered': 1,
+        'failed': 0
+    }),
+    ('True', {
+        'requested': 1,
+        'delivered': 0,
+        'failed': 0
+    })
+    ], ids=['seven_days', 'today']
+)
+def test_get_detailed_service(notify_db, notify_db_session, notify_api, sample_service, today_only, stats):
     with notify_api.test_request_context(), notify_api.test_client() as client:
-        resp = client.get(
+        with freeze_time('2000-01-01T12:00:00'):
-            '/service/{}?detailed=False'.format(sample_service.id),
+            create_sample_notification(notify_db, notify_db_session, status='delivered')
-            headers=[create_authorization_header()]
+        with freeze_time('2000-01-02T12:00:00'):
-        )
+            create_sample_notification(notify_db, notify_db_session, status='created')
+            resp = client.get(
+                '/service/{}?detailed=True&today_only={}'.format(sample_service.id, today_only),
+                headers=[create_authorization_header()]
+            )
 
     assert resp.status_code == 200
     service = json.loads(resp.get_data(as_text=True))['data']
     assert service['id'] == str(sample_service.id)
     assert 'statistics' in service.keys()
     assert set(service['statistics'].keys()) == set(['sms', 'email'])
-    assert service['statistics']['sms'] == {
+    assert service['statistics']['sms'] == stats
-        'requested': 1,
-        'delivered': 0,
-        'failed': 0
-    }
 
 
 # email_counts and sms_counts are 3-tuple of requested, delivered, failed