Merge branch 'master' into remove-contested-writes

app/dao/jobs_dao.py

@@ -1,9 +1,24 @@
-from datetime import date, timedelta
-
 from sqlalchemy import desc, cast, Date as sql_date
 from app import db
 from app.dao import days_ago
-from app.models import Job
+from app.models import Job, NotificationHistory
+from app.statsd_decorators import statsd
+from sqlalchemy import func, asc
+
+
+@statsd(namespace="dao")
+def dao_get_notification_outcomes_for_job(service_id, job_id):
+    query = db.session.query(
+        func.count(NotificationHistory.status).label('count'),
+        NotificationHistory.status.label('status')
+    )
+
+    return query \
+        .filter(NotificationHistory.service_id == service_id) \
+        .filter(NotificationHistory.job_id == job_id)\
+        .group_by(NotificationHistory.status) \
+        .order_by(asc(NotificationHistory.status)) \
+        .all()
 
 
 def dao_get_job_by_service_id_and_job_id(service_id, job_id):

app/job/rest.py

@@ -8,7 +8,8 @@ from flask import (
 from app.dao.jobs_dao import (
     dao_create_job,
     dao_get_job_by_service_id_and_job_id,
-    dao_get_jobs_by_service_id
+    dao_get_jobs_by_service_id,
+    dao_get_notification_outcomes_for_job
 )
 
 from app.dao.services_dao import (
@@ -42,7 +43,11 @@ register_errors(job)
 @job.route('/<job_id>', methods=['GET'])
 def get_job_by_service_and_job_id(service_id, job_id):
     job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
+    statistics = dao_get_notification_outcomes_for_job(service_id, job_id)
     data = job_schema.dump(job).data
+
+    data['statistics'] = [{'status': statistic[1], 'count': statistic[0]} for statistic in statistics]
+
     return jsonify(data=data)
 
 

app/models.py

@@ -303,6 +303,19 @@ class ProviderDetails(db.Model):
 JOB_STATUS_TYPES = ['pending', 'in progress', 'finished', 'sending limits exceeded']
 
 
+JOB_STATUS_PENDING = 'pending'
+JOB_STATUS_IN_PROGRESS = 'in progress'
+JOB_STATUS_FINISHED = 'finished'
+JOB_STATUS_SENDING_LIMITS_EXCEEDED = 'sending limits exceeded'
+JOB_STATUS_SCHEDULED = 'scheduled'
+
+
+class JobStatusTypes(db.Model):
+    __tablename__ = 'job_status'
+
+    name = db.Column(db.String(255), primary_key=True)
+
+
 class Job(db.Model):
     __tablename__ = 'jobs'
 
@@ -343,6 +356,13 @@ class Job(db.Model):
         nullable=True)
     created_by = db.relationship('User')
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
+    scheduled_for = db.Column(
+        db.DateTime,
+        index=True,
+        unique=False,
+        nullable=True)
+    job_status = db.Column(
+        db.String(255), db.ForeignKey('job_status.name'), index=True, nullable=True)
 
 
 VERIFY_CODE_TYPES = [EMAIL_TYPE, SMS_TYPE]

app/schemas.py

@@ -210,7 +210,11 @@ class JobSchema(BaseSchema):
 
     class Meta:
         model = models.Job
-        exclude = ('notifications',)
+        exclude = (
+            'notifications',
+            'notifications_sent',
+            'notifications_delivered',
+            'notifications_failed')
         strict = True
 
 

migrations/versions/0044_jobs_to_notification_hist.py

@@ -20,59 +20,68 @@ from app.models import Job, Template, NotificationHistory
 
 
 def upgrade():
-    session = Session(bind=op.get_bind())
+    #
-
+    # REMOVED
-    go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
+    # This script has been applied and doesn't need to be re-applied
-    notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
+    # note that by referencing the model objects in migration files, any subsequent alteration of the model and thus
-    jobs = session.query(Job).join(Template).filter(Job.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
+    # the database causes all previous migration scripts to fail as the model and DB will be inconsistent in this
-                                                    Job.created_at >= go_live,
+    # past state.
-                                                    Job.created_at < notifications_history_start_date).all()
+    #
-
+    # session = Session(bind=op.get_bind())
-    for job in jobs:
+    #
-        for i in range(0, job.notifications_delivered):
+    # go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
-            notification = NotificationHistory(id=uuid.uuid4(),
+    # notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
-                                               job_id=job.id,
+    # jobs = session.query(Job).join(Template).filter(Job.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
-                                               service_id=job.service_id,
+    #                                                 Job.created_at >= go_live,
-                                               template_id=job.template.id,
+    #                                                 Job.created_at < notifications_history_start_date).all()
-                                               template_version=job.template_version,
+    #
-                                               key_type='normal',
+    # for job in jobs:
-                                               content_char_count=len(job.template.content),
+    #     for i in range(0, job.notifications_delivered):
-                                               notification_type=job.template.template_type,
+    #         notification = NotificationHistory(id=uuid.uuid4(),
-                                               created_at=job.created_at,
+    #                                            job_id=job.id,
-                                               sent_at=job.processing_finished,
+    #                                            service_id=job.service_id,
-                                               sent_by='ses' if job.template.template_type == 'email' else 'mmg',
+    #                                            template_id=job.template.id,
-                                               status='delivered')
+    #                                            template_version=job.template_version,
-
+    #                                            key_type='normal',
-            session.add(notification)
+    #                                            content_char_count=len(job.template.content),
-
+    #                                            notification_type=job.template.template_type,
-        for i in range(0, job.notifications_failed):
+    #                                            created_at=job.created_at,
-            notification = NotificationHistory(id=uuid.uuid4(),
+    #                                            sent_at=job.processing_finished,
-                                               job_id=job.id,
+    #                                            sent_by='ses' if job.template.template_type == 'email' else 'mmg',
-                                               service_id=job.service_id,
+    #                                            status='delivered')
-                                               template_id=job.template.id,
+    #
-                                               template_version=job.template_version,
+    #         session.add(notification)
-                                               key_type='normal',
+    #
-                                               content_char_count=len(job.template.content),
+    #     for i in range(0, job.notifications_failed):
-                                               notification_type=job.template.template_type,
+    #         notification = NotificationHistory(id=uuid.uuid4(),
-                                               created_at=job.created_at,
+    #                                            job_id=job.id,
-                                               sent_at=job.processing_finished,
+    #                                            service_id=job.service_id,
-                                               sent_by='ses' if job.template.template_type == 'email' else 'mmg',
+    #                                            template_id=job.template.id,
-                                               status='permanent-failure')
+    #                                            template_version=job.template_version,
-            session.add(notification)
+    #                                            key_type='normal',
-        session.commit()
+    #                                            content_char_count=len(job.template.content),
+    #                                            notification_type=job.template.template_type,
+    #                                            created_at=job.created_at,
+    #                                            sent_at=job.processing_finished,
+    #                                            sent_by='ses' if job.template.template_type == 'email' else 'mmg',
+    #                                            status='permanent-failure')
+    #         session.add(notification)
+    #     session.commit()
+    pass
 
 
 def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
+    # ### commands auto generated by Alembic - please adjust! ###
-    session = Session(bind=op.get_bind())
+    # session = Session(bind=op.get_bind())
-
+    #
-    go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
+    # go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d')
-    notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
+    # notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S')
-
+    #
-    session.query(NotificationHistory).filter(
+    # session.query(NotificationHistory).filter(
-        NotificationHistory.created_at >= go_live,
+    #     NotificationHistory.created_at >= go_live,
-        NotificationHistory.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
+    #     NotificationHistory.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091',
-        NotificationHistory.created_at < notifications_history_start_date).delete()
+    #     NotificationHistory.created_at < notifications_history_start_date).delete()
-
+    #
-    session.commit()
+    # session.commit()
-    ### end Alembic commands ###
+    # ### end Alembic commands ###
+    pass

migrations/versions/0048_job_scheduled_time.py

@@ -0,0 +1,41 @@
+"""empty message
+
+Revision ID: 0048_job_scheduled_time
+Revises: 0047_ukvi_spelling
+Create Date: 2016-08-24 13:21:51.744526
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0048_job_scheduled_time'
+down_revision = '0047_ukvi_spelling'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('job_status',
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('name')
+    )
+    op.add_column('jobs', sa.Column('job_status', sa.String(length=255), nullable=True))
+    op.add_column('jobs', sa.Column('scheduled_for', sa.DateTime(), nullable=True))
+    op.create_index(op.f('ix_jobs_job_status'), 'jobs', ['job_status'], unique=False)
+    op.create_index(op.f('ix_jobs_scheduled_for'), 'jobs', ['scheduled_for'], unique=False)
+    op.create_foreign_key(None, 'jobs', 'job_status', ['job_status'], ['name'])
+
+    op.execute("insert into job_status values ('pending')")
+    op.execute("insert into job_status values ('in progress')")
+    op.execute("insert into job_status values ('finished')")
+    op.execute("insert into job_status values ('sending limits exceeded')")
+    op.execute("insert into job_status values ('scheduled')")
+
+
+def downgrade():
+    op.drop_constraint('jobs_job_status_fkey', 'jobs', type_='foreignkey')
+    op.drop_index(op.f('ix_jobs_scheduled_for'), table_name='jobs')
+    op.drop_index(op.f('ix_jobs_job_status'), table_name='jobs')
+    op.drop_column('jobs', 'scheduled_for')
+    op.drop_column('jobs', 'job_status')
+    op.drop_table('job_status')

tests/app/dao/test_jobs_dao.py

@@ -5,10 +5,95 @@ from app.dao.jobs_dao import (
     dao_get_job_by_service_id_and_job_id,
     dao_create_job,
     dao_update_job,
-    dao_get_jobs_by_service_id
+    dao_get_jobs_by_service_id,
-)
+    dao_get_notification_outcomes_for_job)
 
 from app.models import Job
+from tests.app.conftest import sample_notification, sample_job, sample_service
+
+
+def test_should_have_decorated_notifications_dao_functions():
+    assert dao_get_notification_outcomes_for_job.__wrapped__.__name__ == 'dao_get_notification_outcomes_for_job'  # noqa
+
+
+def test_should_get_all_statuses_for_notifications_associated_with_job(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_job):
+
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='delivered')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='pending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='failed')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='technical-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='temporary-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='permanent-failure')  # noqa
+
+    results = dao_get_notification_outcomes_for_job(sample_service.id, sample_job.id)
+    assert [(row.count, row.status) for row in results] == [
+        (1, 'created'),
+        (1, 'sending'),
+        (1, 'delivered'),
+        (1, 'pending'),
+        (1, 'failed'),
+        (1, 'technical-failure'),
+        (1, 'temporary-failure'),
+        (1, 'permanent-failure')
+    ]
+
+
+def test_should_count_of_statuses_for_notifications_associated_with_job(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_job):
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='delivered')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=sample_job, status='delivered')
+
+    results = dao_get_notification_outcomes_for_job(sample_service.id, sample_job.id)
+    assert [(row.count, row.status) for row in results] == [
+        (2, 'created'),
+        (4, 'sending'),
+        (2, 'delivered')
+    ]
+
+
+def test_should_return_zero_length_array_if_no_notifications_for_job(sample_service, sample_job):
+    assert len(dao_get_notification_outcomes_for_job(sample_job.id, sample_service.id)) == 0
+
+
+def test_should_return_notifications_only_for_this_job(notify_db, notify_db_session, sample_service):
+    job_1 = sample_job(notify_db, notify_db_session, service=sample_service)
+    job_2 = sample_job(notify_db, notify_db_session, service=sample_service)
+
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=job_1, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_service, job=job_2, status='created')
+
+    results = dao_get_notification_outcomes_for_job(sample_service.id, job_1.id)
+    assert [(row.count, row.status) for row in results] == [
+        (1, 'created')
+    ]
+
+
+def test_should_return_notifications_only_for_this_service(notify_db, notify_db_session):
+    service_1 = sample_service(notify_db, notify_db_session, service_name="one", email_from="one")
+    service_2 = sample_service(notify_db, notify_db_session, service_name="two", email_from="two")
+
+    job_1 = sample_job(notify_db, notify_db_session, service=service_1)
+    job_2 = sample_job(notify_db, notify_db_session, service=service_2)
+
+    sample_notification(notify_db, notify_db_session, service=service_1, job=job_1, status='created')
+    sample_notification(notify_db, notify_db_session, service=service_2, job=job_2, status='created')
+
+    assert len(dao_get_notification_outcomes_for_job(service_1.id, job_2.id)) == 0
 
 
 def test_create_job(sample_template):

tests/app/job/test_rest.py

@@ -9,7 +9,7 @@ import app.celery.tasks
 from tests import create_authorization_header
 from tests.app.conftest import (
     sample_job as create_job,
-    sample_notification as create_sample_notification)
+    sample_notification as create_sample_notification, sample_notification)
 from app.dao.templates_dao import dao_update_template
 from app.models import NOTIFICATION_STATUS_TYPES
 
@@ -94,20 +94,6 @@ def test_get_job_with_unknown_id_returns404(notify_api, sample_template, fake_uu
             }
 
 
-def test_get_job_by_id(notify_api, sample_job):
-    job_id = str(sample_job.id)
-    service_id = sample_job.service.id
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            path = '/service/{}/job/{}'.format(service_id, job_id)
-            auth_header = create_authorization_header(service_id=sample_job.service.id)
-            response = client.get(path, headers=[auth_header])
-            assert response.status_code == 200
-            resp_json = json.loads(response.get_data(as_text=True))
-            assert resp_json['data']['id'] == job_id
-            assert resp_json['data']['created_by']['name'] == 'Test User'
-
-
 def test_create_job(notify_api, sample_template, mocker, fake_uuid):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
@@ -123,6 +109,7 @@ def test_create_job(notify_api, sample_template, mocker, fake_uuid):
             path = '/service/{}/job'.format(sample_template.service.id)
             auth_header = create_authorization_header(service_id=sample_template.service.id)
             headers = [('Content-Type', 'application/json'), auth_header]
+
             response = client.post(
                 path,
                 data=json.dumps(data),
@@ -350,3 +337,83 @@ def test_get_all_notifications_for_job_filtered_by_status(
         resp = json.loads(response.get_data(as_text=True))
         assert len(resp['notifications']) == expected_notification_count
         assert response.status_code == 200
+
+
+def test_get_job_by_id(notify_api, sample_job):
+    job_id = str(sample_job.id)
+    service_id = sample_job.service.id
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, job_id)
+            auth_header = create_authorization_header(service_id=sample_job.service.id)
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            assert resp_json['data']['id'] == job_id
+            assert resp_json['data']['statistics'] == []
+            assert resp_json['data']['created_by']['name'] == 'Test User'
+
+
+def test_get_job_by_id_should_return_statistics(notify_db, notify_db_session, notify_api, sample_job):
+    job_id = str(sample_job.id)
+    service_id = sample_job.service.id
+
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='delivered')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='pending')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='failed')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='technical-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='temporary-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='permanent-failure')  # noqa
+
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, job_id)
+            auth_header = create_authorization_header(service_id=sample_job.service.id)
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            print(resp_json)
+            assert resp_json['data']['id'] == job_id
+            assert {'status': 'created', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'sending', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'delivered', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'pending', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'failed', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'technical-failure', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'temporary-failure', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'permanent-failure', 'count': 1} in resp_json['data']['statistics']
+            assert resp_json['data']['created_by']['name'] == 'Test User'
+
+
+def test_get_job_by_id_should_return_summed_statistics(notify_db, notify_db_session, notify_api, sample_job):
+    job_id = str(sample_job.id)
+    service_id = sample_job.service.id
+
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='created')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='sending')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='failed')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='failed')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='failed')
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='technical-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='temporary-failure')  # noqa
+    sample_notification(notify_db, notify_db_session, service=sample_job.service, job=sample_job, status='temporary-failure')  # noqa
+
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            path = '/service/{}/job/{}'.format(service_id, job_id)
+            auth_header = create_authorization_header(service_id=sample_job.service.id)
+            response = client.get(path, headers=[auth_header])
+            assert response.status_code == 200
+            resp_json = json.loads(response.get_data(as_text=True))
+            print(resp_json)
+            assert resp_json['data']['id'] == job_id
+            assert {'status': 'created', 'count': 3} in resp_json['data']['statistics']
+            assert {'status': 'sending', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'failed', 'count': 3} in resp_json['data']['statistics']
+            assert {'status': 'technical-failure', 'count': 1} in resp_json['data']['statistics']
+            assert {'status': 'temporary-failure', 'count': 2} in resp_json['data']['statistics']
+            assert resp_json['data']['created_by']['name'] == 'Test User'