Merge pull request #2670 from alphagov/uploads-endpoint

Uploads endpoint

app/__init__.py

@@ -124,6 +124,7 @@ def register_blueprint(application):
     from app.platform_stats.rest import platform_stats_blueprint
     from app.template_folder.rest import template_folder_blueprint
     from app.letter_branding.letter_branding_rest import letter_branding_blueprint
+    from app.upload.rest import upload_blueprint
 
     service_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -206,6 +207,9 @@ def register_blueprint(application):
     letter_branding_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(letter_branding_blueprint)
 
+    upload_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(upload_blueprint)
+
 
 def register_v2_blueprints(application):
     from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms

app/dao/uploads_dao.py

@@ -0,0 +1,68 @@
+from flask import current_app
+from sqlalchemy import desc, literal
+
+from app import db
+from app.models import (
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED
+)
+from app.utils import midnight_n_days_ago
+
+
+def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50):
+    # Hardcoded filter to exclude cancelled or scheduled jobs
+    # for the moment, but we may want to change this method take 'statuses' as a argument in the future
+    jobs_query_filter = [
+        Job.service_id == service_id,
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
+        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED])
+    ]
+    if limit_days is not None:
+        jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
+
+    jobs_query = db.session.query(
+        Job.id,
+        Job.original_file_name,
+        Job.notification_count,
+        Job.created_at.label("created_at"),
+        Job.scheduled_for.label("scheduled_for"),
+        Job.processing_started.label('processing_started'),
+        Job.job_status.label("status"),
+        literal('job').label('upload_type')
+    ).filter(
+        *jobs_query_filter
+    )
+
+    letters_query_filter = [
+        Notification.service_id == service_id,
+        Notification.notification_type == LETTER_TYPE,
+        Notification.api_key_id == None,  # noqa
+        Notification.status != NOTIFICATION_CANCELLED,
+        Template.hidden == True,
+
+    ]
+    if limit_days is not None:
+        letters_query_filter.append(Notification.created_at >= midnight_n_days_ago(limit_days))
+
+    letters_query = db.session.query(
+        Notification.id,
+        Notification.client_reference.label('original_file_name'),
+        literal('1').label('notification_count'),
+        Notification.created_at.label("created_at"),
+        literal(None).label('scheduled_for'),
+        # letters don't have a processing_started date but we want created_at to be used for sorting
+        Notification.created_at.label('processing_started'),
+        Notification.status,
+        literal('letter').label('upload_type')
+    ).join(
+        Template, Notification.template_id == Template.id
+    ).filter(
+        *letters_query_filter
+    )
+
+    return jobs_query.union_all(
+        letters_query
+    ).order_by(
+        desc("processing_started"), desc("created_at")
+    ).paginate(page=page, per_page=page_size)

app/upload/rest.py

@@ -0,0 +1,74 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    request,
+    current_app
+)
+
+from app.dao.fact_notification_status_dao import fetch_notification_statuses_for_job
+from app.dao.jobs_dao import (
+    dao_get_notification_outcomes_for_job
+)
+from app.dao.uploads_dao import dao_get_uploads_by_service_id
+from app.errors import (
+    register_errors,
+)
+from app.utils import pagination_links, midnight_n_days_ago
+
+upload_blueprint = Blueprint('upload', __name__, url_prefix='/service/<uuid:service_id>/upload')
+
+register_errors(upload_blueprint)
+
+
+@upload_blueprint.route('', methods=['GET'])
+def get_uploads_by_service(service_id):
+    return jsonify(**get_paginated_uploads(service_id,
+                                           request.args.get('limit_days', type=int),
+                                           request.args.get('page', type=int)))
+
+
+def get_paginated_uploads(service_id, limit_days, page):
+    pagination = dao_get_uploads_by_service_id(
+        service_id,
+        limit_days=limit_days,
+        page=page,
+        page_size=current_app.config['PAGE_SIZE']
+    )
+    uploads = pagination.items
+    data = []
+    for upload in uploads:
+        upload_dict = {
+            'id': upload.id,
+            'original_file_name': upload.original_file_name,
+            'notification_count': upload.notification_count,
+            'created_at': upload.scheduled_for.strftime(
+                "%Y-%m-%d %H:%M:%S") if upload.scheduled_for else upload.created_at.strftime("%Y-%m-%d %H:%M:%S"),
+            'upload_type': upload.upload_type
+        }
+        if upload.upload_type == 'job':
+            start = upload.processing_started
+
+            if start is None:
+                statistics = []
+            elif start.replace(tzinfo=None) < midnight_n_days_ago(3):
+                # ft_notification_status table
+                statistics = fetch_notification_statuses_for_job(upload.id)
+            else:
+                # notifications table
+                statistics = dao_get_notification_outcomes_for_job(service_id, upload.id)
+            upload_dict['statistics'] = [{'status': statistic.status, 'count': statistic.count} for statistic in
+                                         statistics]
+        else:
+            upload_dict['statistics'] = [{'status': upload.status, 'count': 1}]
+        data.append(upload_dict)
+
+    return {
+        'data': data,
+        'page_size': pagination.per_page,
+        'total': pagination.total,
+        'links': pagination_links(
+            pagination,
+            '.get_uploads_by_service',
+            service_id=service_id
+        )
+    }

tests/app/dao/test_uploads_dao.py

@@ -0,0 +1,168 @@
+from datetime import datetime, timedelta
+
+from app.dao.uploads_dao import dao_get_uploads_by_service_id
+from app.models import LETTER_TYPE, JOB_STATUS_IN_PROGRESS
+from tests.app.db import create_job, create_service, create_template, create_notification
+
+
+def create_uploaded_letter(letter_template, service, status='created', created_at=None):
+    return create_notification(
+        template=letter_template,
+        to_field="file-name",
+        status=status,
+        reference="dvla-reference",
+        client_reference="file-name",
+        one_off=True,
+        created_by_id=service.users[0].id,
+        created_at=created_at
+    )
+
+
+def create_uploaded_template(service):
+    return create_template(
+        service,
+        template_type=LETTER_TYPE,
+        template_name='Pre-compiled PDF',
+        subject='Pre-compiled PDF',
+        content="",
+        hidden=True,
+        postage="second",
+    )
+
+
+def test_get_uploads_for_service(sample_template):
+    job = create_job(sample_template, processing_started=datetime.utcnow())
+    letter_template = create_uploaded_template(sample_template.service)
+    letter = create_uploaded_letter(letter_template, sample_template.service)
+
+    other_service = create_service(service_name="other service")
+    other_template = create_template(service=other_service)
+    other_job = create_job(other_template, processing_started=datetime.utcnow())
+    other_letter_template = create_uploaded_template(other_service)
+    other_letter = create_uploaded_letter(other_letter_template, other_service)
+
+    uploads_from_db = dao_get_uploads_by_service_id(job.service_id).items
+    other_uploads_from_db = dao_get_uploads_by_service_id(other_job.service_id).items
+
+    assert len(uploads_from_db) == 2
+
+    assert uploads_from_db[0] == (letter.id, letter.client_reference, 1, letter.created_at,
+                                  None, letter.created_at, letter.status, "letter")
+    assert uploads_from_db[1] == (job.id, job.original_file_name, job.notification_count, job.created_at,
+                                  job.scheduled_for, job.processing_started, job.job_status, "job")
+
+    assert len(other_uploads_from_db) == 2
+    assert other_uploads_from_db[0] == (other_letter.id,
+                                        other_letter.client_reference,
+                                        1,
+                                        other_letter.created_at,
+                                        None,
+                                        other_letter.created_at,
+                                        other_letter.status,
+                                        "letter")
+    assert other_uploads_from_db[1] == (other_job.id,
+                                        other_job.original_file_name,
+                                        other_job.notification_count,
+                                        other_job.created_at,
+                                        other_job.scheduled_for,
+                                        other_job.processing_started,
+                                        other_job.job_status, "job")
+
+    assert uploads_from_db[0] != other_uploads_from_db[0]
+    assert uploads_from_db[1] != other_uploads_from_db[1]
+
+
+def test_get_uploads_does_not_return_cancelled_jobs_or_letters(sample_template):
+    create_job(sample_template, job_status='scheduled')
+    create_job(sample_template, job_status='cancelled')
+    letter_template = create_uploaded_template(sample_template.service)
+    create_uploaded_letter(letter_template, sample_template.service, status='cancelled')
+
+    assert len(dao_get_uploads_by_service_id(sample_template.service_id).items) == 0
+
+
+def test_get_uploads_orders_by_created_at_desc(sample_template):
+    letter_template = create_uploaded_template(sample_template.service)
+
+    upload_1 = create_job(sample_template, processing_started=datetime.utcnow(),
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_2 = create_job(sample_template, processing_started=datetime.utcnow(),
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered')
+
+    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
+
+    assert len(results) == 3
+    assert results[0].id == upload_3.id
+    assert results[1].id == upload_2.id
+    assert results[2].id == upload_1.id
+
+
+def test_get_uploads_orders_by_processing_started_desc(sample_template):
+    days_ago = datetime.utcnow() - timedelta(days=3)
+    upload_1 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1),
+                          created_at=days_ago,
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=2),
+                          created_at=days_ago,
+                          job_status=JOB_STATUS_IN_PROGRESS)
+
+    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
+
+    assert len(results) == 2
+    assert results[0].id == upload_1.id
+    assert results[1].id == upload_2.id
+
+
+def test_get_uploads_orders_by_processing_started_and_created_at_desc(sample_template):
+    letter_template = create_uploaded_template(sample_template.service)
+
+    days_ago = datetime.utcnow() - timedelta(days=4)
+    upload_1 = create_uploaded_letter(letter_template, service=letter_template.service)
+    upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1),
+                          created_at=days_ago,
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_3 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=2),
+                          created_at=days_ago,
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_4 = create_uploaded_letter(letter_template, service=letter_template.service,
+                                      created_at=datetime.utcnow() - timedelta(days=3))
+
+    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
+
+    assert len(results) == 4
+    assert results[0].id == upload_1.id
+    assert results[1].id == upload_2.id
+    assert results[2].id == upload_3.id
+    assert results[3].id == upload_4.id
+
+
+def test_get_uploads_is_paginated(sample_template):
+    letter_template = create_uploaded_template(sample_template.service)
+
+    upload_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered',
+                                      created_at=datetime.utcnow() - timedelta(minutes=3))
+    upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(minutes=2),
+                          job_status=JOB_STATUS_IN_PROGRESS)
+    upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered',
+                                      created_at=datetime.utcnow() - timedelta(minutes=1))
+    upload_4 = create_job(sample_template, processing_started=datetime.utcnow(), job_status=JOB_STATUS_IN_PROGRESS)
+
+    results = dao_get_uploads_by_service_id(sample_template.service_id, page=1, page_size=2)
+
+    assert results.per_page == 2
+    assert results.total == 4
+    assert len(results.items) == 2
+    assert results.items[0].id == upload_4.id
+    assert results.items[1].id == upload_3.id
+
+    results = dao_get_uploads_by_service_id(sample_template.service_id, page=2, page_size=2)
+
+    assert len(results.items) == 2
+    assert results.items[0].id == upload_2.id
+    assert results.items[1].id == upload_1.id
+
+
+def test_get_uploads_returns_empty_list(sample_service):
+    items = dao_get_uploads_by_service_id(sample_service.id).items
+    assert items == []

tests/app/upload/test_rest.py

@@ -0,0 +1,178 @@
+from datetime import datetime, timedelta, date
+
+from freezegun import freeze_time
+
+from app.models import LETTER_TYPE, JOB_STATUS_FINISHED, JOB_STATUS_PENDING
+from tests.app.db import create_job, create_notification, create_template, create_ft_notification_status
+from tests.conftest import set_config
+
+
+def create_uploaded_letter(letter_template, service, status='created', created_at=None):
+    return create_notification(
+        template=letter_template,
+        to_field="file-name",
+        status=status,
+        reference="dvla-reference",
+        client_reference="file-name",
+        one_off=True,
+        created_by_id=service.users[0].id,
+        created_at=created_at
+    )
+
+
+def create_precompiled_template(service):
+    return create_template(
+        service,
+        template_type=LETTER_TYPE,
+        template_name='Pre-compiled PDF',
+        subject='Pre-compiled PDF',
+        content="",
+        hidden=True,
+        postage="second",
+    )
+
+
+def test_get_uploads(admin_request, sample_template):
+    letter_template = create_precompiled_template(sample_template.service)
+
+    upload_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered',
+                                      created_at=datetime.utcnow() - timedelta(minutes=4))
+    upload_2 = create_job(template=sample_template,
+                          processing_started=datetime.utcnow() - timedelta(minutes=3),
+                          job_status=JOB_STATUS_FINISHED)
+    upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered',
+                                      created_at=datetime.utcnow() - timedelta(minutes=2))
+    upload_4 = create_job(template=sample_template,
+                          processing_started=datetime.utcnow() - timedelta(minutes=1),
+                          job_status=JOB_STATUS_FINISHED)
+    upload_5 = create_job(template=sample_template, processing_started=None,
+                          job_status=JOB_STATUS_PENDING, notification_count=10)
+
+    service_id = sample_template.service.id
+
+    resp_json = admin_request.get('upload.get_uploads_by_service', service_id=service_id)
+    data = resp_json['data']
+    assert len(data) == 5
+    assert data[0] == {'id': str(upload_5.id),
+                       'original_file_name': 'some.csv',
+                       'notification_count': 10,
+                       'created_at': upload_5.created_at.strftime("%Y-%m-%d %H:%M:%S"),
+                       'statistics': [],
+                       'upload_type': 'job'}
+    assert data[1] == {'id': str(upload_4.id),
+                       'original_file_name': 'some.csv',
+                       'notification_count': 1,
+                       'created_at': upload_4.created_at.strftime(
+                           "%Y-%m-%d %H:%M:%S"),
+                       'statistics': [],
+                       'upload_type': 'job'}
+    assert data[2] == {'id': str(upload_3.id),
+                       'original_file_name': "file-name",
+                       'notification_count': 1,
+                       'created_at': upload_3.created_at.strftime("%Y-%m-%d %H:%M:%S"),
+                       'statistics': [{'count': 1, 'status': 'delivered'}],
+                       'upload_type': 'letter'}
+    assert data[3] == {'id': str(upload_2.id),
+                       'original_file_name': "some.csv",
+                       'notification_count': 1,
+                       'created_at': upload_2.created_at.strftime(
+                           "%Y-%m-%d %H:%M:%S"),
+                       'statistics': [],
+                       'upload_type': 'job'}
+    assert data[4] == {'id': str(upload_1.id),
+                       'original_file_name': "file-name",
+                       'notification_count': 1,
+                       'created_at': upload_1.created_at.strftime("%Y-%m-%d %H:%M:%S"),
+                       'statistics': [{'count': 1, 'status': 'delivered'}],
+                       'upload_type': 'letter'}
+
+
+def test_get_uploads_should_return_statistics(admin_request, sample_template):
+    now = datetime.utcnow()
+    earlier = datetime.utcnow() - timedelta(days=1)
+    job_1 = create_job(template=sample_template, job_status='pending')
+    job_2 = create_job(sample_template, processing_started=earlier)
+    for _ in range(3):
+        create_notification(template=sample_template, job=job_2, status='created')
+
+    job_3 = create_job(sample_template, processing_started=now)
+    for _ in range(4):
+        create_notification(template=sample_template, job=job_3, status='sending')
+
+    letter_template = create_precompiled_template(sample_template.service)
+    letter_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered',
+                                      created_at=datetime.utcnow() - timedelta(days=3))
+
+    resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id)['data']
+    assert len(resp_json) == 4
+    assert resp_json[0]['id'] == str(job_1.id)
+    assert resp_json[0]['statistics'] == []
+    assert resp_json[1]['id'] == str(job_3.id)
+    assert resp_json[1]['statistics'] == [{'status': 'sending', 'count': 4}]
+    assert resp_json[2]['id'] == str(job_2.id)
+    assert resp_json[2]['statistics'] == [{'status': 'created', 'count': 3}]
+    assert resp_json[3]['id'] == str(letter_1.id)
+    assert resp_json[3]['statistics'] == [{'status': 'delivered', 'count': 1}]
+
+
+def test_get_uploads_should_paginate(admin_request, sample_template):
+    for _ in range(10):
+        create_job(sample_template)
+
+    with set_config(admin_request.app, 'PAGE_SIZE', 2):
+        resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id)
+
+    assert len(resp_json['data']) == 2
+    assert resp_json['page_size'] == 2
+    assert resp_json['total'] == 10
+    assert 'links' in resp_json
+    assert set(resp_json['links'].keys()) == {'next', 'last'}
+
+
+def test_get_uploads_accepts_page_parameter(admin_request, sample_template):
+    for _ in range(10):
+        create_job(sample_template)
+
+    with set_config(admin_request.app, 'PAGE_SIZE', 2):
+        resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id, page=2)
+
+    assert len(resp_json['data']) == 2
+    assert resp_json['page_size'] == 2
+    assert resp_json['total'] == 10
+    assert 'links' in resp_json
+    assert set(resp_json['links'].keys()) == {'prev', 'next', 'last'}
+
+
+@freeze_time('2017-06-10 12:00')
+def test_get_uploads_should_retrieve_from_ft_notification_status_for_old_jobs(admin_request, sample_template):
+    # it's the 10th today, so 3 days should include all of 7th, 8th, 9th, and some of 10th.
+    just_three_days_ago = datetime(2017, 6, 6, 22, 59, 59)
+    not_quite_three_days_ago = just_three_days_ago + timedelta(seconds=1)
+
+    job_1 = create_job(sample_template, created_at=just_three_days_ago, processing_started=just_three_days_ago)
+    job_2 = create_job(sample_template, created_at=just_three_days_ago, processing_started=not_quite_three_days_ago)
+    # is old but hasn't started yet (probably a scheduled job). We don't have any stats for this job yet.
+    job_3 = create_job(sample_template, created_at=just_three_days_ago, processing_started=None)
+
+    # some notifications created more than three days ago, some created after the midnight cutoff
+    create_ft_notification_status(date(2017, 6, 6), job=job_1, notification_status='delivered', count=2)
+    create_ft_notification_status(date(2017, 6, 7), job=job_1, notification_status='delivered', count=4)
+    # job2's new enough
+    create_notification(job=job_2, status='created', created_at=not_quite_three_days_ago)
+
+    # this isn't picked up because the job is too new
+    create_ft_notification_status(date(2017, 6, 7), job=job_2, notification_status='delivered', count=8)
+    # this isn't picked up - while the job is old, it started in last 3 days so we look at notification table instead
+    create_ft_notification_status(date(2017, 6, 7), job=job_3, notification_status='delivered', count=16)
+
+    # this isn't picked up because we're using the ft status table for job_1 as it's old
+    create_notification(job=job_1, status='created', created_at=not_quite_three_days_ago)
+
+    resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id)['data']
+
+    assert resp_json[0]['id'] == str(job_3.id)
+    assert resp_json[0]['statistics'] == []
+    assert resp_json[1]['id'] == str(job_2.id)
+    assert resp_json[1]['statistics'] == [{'status': 'created', 'count': 1}]
+    assert resp_json[2]['id'] == str(job_1.id)
+    assert resp_json[2]['statistics'] == [{'status': 'delivered', 'count': 6}]