diff --git a/app/__init__.py b/app/__init__.py index 141e45582..077d68ec2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -124,6 +124,7 @@ def register_blueprint(application): from app.platform_stats.rest import platform_stats_blueprint from app.template_folder.rest import template_folder_blueprint from app.letter_branding.letter_branding_rest import letter_branding_blueprint + from app.upload.rest import upload_blueprint service_blueprint.before_request(requires_admin_auth) application.register_blueprint(service_blueprint, url_prefix='/service') @@ -206,6 +207,9 @@ def register_blueprint(application): letter_branding_blueprint.before_request(requires_admin_auth) application.register_blueprint(letter_branding_blueprint) + upload_blueprint.before_request(requires_admin_auth) + application.register_blueprint(upload_blueprint) + def register_v2_blueprints(application): from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms diff --git a/app/dao/uploads_dao.py b/app/dao/uploads_dao.py new file mode 100644 index 000000000..2dad620da --- /dev/null +++ b/app/dao/uploads_dao.py @@ -0,0 +1,66 @@ +from flask import current_app +from sqlalchemy import desc, literal + +from app import db +from app.models import ( + Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED, + NOTIFICATION_CANCELLED +) +from app.utils import midnight_n_days_ago + + +def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50): + jobs_query_filter = [ + Job.service_id == service_id, + Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'], + Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'], + Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]) + ] + if limit_days is not None: + jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days)) + + jobs_query = db.session.query( + Job.id, + Job.original_file_name, + Job.notification_count, + Job.created_at.label("created_at"), + Job.scheduled_for.label("scheduled_for"), + Job.processing_started.label('processing_started'), + Job.job_status.label("status"), + literal('job').label('upload_type') + ).filter( + *jobs_query_filter + ) + + letters_query_filter = [ + Notification.service_id == service_id, + Notification.notification_type == LETTER_TYPE, + Notification.api_key_id == None, # noqa + Notification.status != NOTIFICATION_CANCELLED, + Template.hidden == True, + + ] + if limit_days is not None: + letters_query_filter.append(Notification.created_at >= midnight_n_days_ago(limit_days)) + + letters_query = db.session.query( + Notification.id, + Notification.client_reference.label('original_file_name'), + literal('1').label('notification_count'), + Notification.created_at.label("created_at"), + literal(None).label('scheduled_for'), + # letters don't have a processing_started date but we want created_at to be used for sorting + Notification.created_at.label('processing_started'), + Notification.status, + literal('letter').label('upload_type') + ).join( + Template, Notification.template_id == Template.id + ).filter( + *letters_query_filter + ) + + return jobs_query.union_all( + letters_query + ).order_by( + desc("processing_started"), desc("created_at") + ).paginate(page=page, per_page=page_size) diff --git a/app/upload/__init__.py b/app/upload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/upload/rest.py b/app/upload/rest.py new file mode 100644 index 000000000..e3958b38e --- /dev/null +++ b/app/upload/rest.py @@ -0,0 +1,83 @@ +from flask import ( + Blueprint, + jsonify, + request, + current_app +) + +from app.dao.fact_notification_status_dao import fetch_notification_statuses_for_job +from app.dao.jobs_dao import ( + dao_get_notification_outcomes_for_job +) +from app.dao.uploads_dao import dao_get_uploads_by_service_id +from app.errors import ( + register_errors, + InvalidRequest +) +from app.utils import pagination_links, midnight_n_days_ago + +upload_blueprint = Blueprint('upload', __name__, url_prefix='/service//upload') + +register_errors(upload_blueprint) + + +@upload_blueprint.route('', methods=['GET']) +def get_uploads_by_service(service_id): + if request.args.get('limit_days'): + try: + limit_days = int(request.args['limit_days']) + except ValueError: + errors = {'limit_days': ['{} is not an integer'.format(request.args['limit_days'])]} + raise InvalidRequest(errors, status_code=400) + else: + limit_days = None + + page = int(request.args.get('page', 1)) + return jsonify(**get_paginated_uploads(service_id, limit_days, page)) + + +def get_paginated_uploads(service_id, limit_days, page): + pagination = dao_get_uploads_by_service_id( + service_id, + limit_days=limit_days, + page=page, + page_size=current_app.config['PAGE_SIZE'] + ) + uploads = pagination.items + data = [] + for upload in uploads: + upload_dict = { + 'id': upload.id, + 'original_file_name': upload.original_file_name, + 'notification_count': upload.notification_count, + 'created_at': upload.scheduled_for.strftime( + "%Y-%m-%d %H:%M:%S") if upload.scheduled_for else upload.created_at.strftime("%Y-%m-%d %H:%M:%S"), + 'upload_type': upload.upload_type + } + if upload.upload_type == 'job': + start = upload.processing_started + + if start is None: + statistics = [] + elif start.replace(tzinfo=None) < midnight_n_days_ago(3): + # ft_notification_status table + statistics = fetch_notification_statuses_for_job(upload.id) + else: + # notifications table + statistics = dao_get_notification_outcomes_for_job(service_id, upload.id) + upload_dict['statistics'] = [{'status': statistic.status, 'count': statistic.count} for statistic in + statistics] + else: + upload_dict['statistics'] = [{'status': upload.status, 'count': 1}] + data.append(upload_dict) + + return { + 'data': data, + 'page_size': pagination.per_page, + 'total': pagination.total, + 'links': pagination_links( + pagination, + '.get_uploads_by_service', + service_id=service_id + ) + } diff --git a/tests/app/dao/test_uploads_dao.py b/tests/app/dao/test_uploads_dao.py new file mode 100644 index 000000000..d6703a801 --- /dev/null +++ b/tests/app/dao/test_uploads_dao.py @@ -0,0 +1,168 @@ +from datetime import datetime, timedelta + +from app.dao.uploads_dao import dao_get_uploads_by_service_id +from app.models import LETTER_TYPE, JOB_STATUS_IN_PROGRESS +from tests.app.db import create_job, create_service, create_template, create_notification + + +def create_uploaded_letter(letter_template, service, status='created', created_at=None): + return create_notification( + template=letter_template, + to_field="file-name", + status=status, + reference="dvla-reference", + client_reference="file-name", + one_off=True, + created_by_id=service.users[0].id, + created_at=created_at + ) + + +def create_precompiled_template(service): + return create_template( + service, + template_type=LETTER_TYPE, + template_name='Pre-compiled PDF', + subject='Pre-compiled PDF', + content="", + hidden=True, + postage="second", + ) + + +def test_get_uploads_for_service(sample_template): + job = create_job(sample_template, processing_started=datetime.utcnow()) + letter_template = create_precompiled_template(sample_template.service) + letter = create_uploaded_letter(letter_template, sample_template.service) + + other_service = create_service(service_name="other service") + other_template = create_template(service=other_service) + other_job = create_job(other_template, processing_started=datetime.utcnow()) + other_letter_template = create_precompiled_template(other_service) + other_letter = create_uploaded_letter(other_letter_template, other_service) + + uploads_from_db = dao_get_uploads_by_service_id(job.service_id).items + other_uploads_from_db = dao_get_uploads_by_service_id(other_job.service_id).items + + assert len(uploads_from_db) == 2 + + assert uploads_from_db[0] == (letter.id, letter.client_reference, 1, letter.created_at, + None, letter.created_at, letter.status, "letter") + assert uploads_from_db[1] == (job.id, job.original_file_name, job.notification_count, job.created_at, + job.scheduled_for, job.processing_started, job.job_status, "job") + + assert len(other_uploads_from_db) == 2 + assert other_uploads_from_db[0] == (other_letter.id, + other_letter.client_reference, + 1, + other_letter.created_at, + None, + other_letter.created_at, + other_letter.status, + "letter") + assert other_uploads_from_db[1] == (other_job.id, + other_job.original_file_name, + other_job.notification_count, + other_job.created_at, + other_job.scheduled_for, + other_job.processing_started, + other_job.job_status, "job") + + assert uploads_from_db[0] != other_uploads_from_db[0] + assert uploads_from_db[1] != other_uploads_from_db[1] + + +def test_get_uploads_does_not_return_cancelled_jobs_or_letters(sample_template): + create_job(sample_template, job_status='scheduled') + create_job(sample_template, job_status='cancelled') + letter_template = create_precompiled_template(sample_template.service) + create_uploaded_letter(letter_template, sample_template.service, status='cancelled') + + assert len(dao_get_uploads_by_service_id(sample_template.service_id).items) == 0 + + +def test_get_uploads_orders_by_created_at_desc(sample_template): + letter_template = create_precompiled_template(sample_template.service) + + upload_1 = create_job(sample_template, processing_started=datetime.utcnow(), + job_status=JOB_STATUS_IN_PROGRESS) + upload_2 = create_job(sample_template, processing_started=datetime.utcnow(), + job_status=JOB_STATUS_IN_PROGRESS) + upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered') + + results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items + + assert len(results) == 3 + assert results[0].id == upload_3.id + assert results[1].id == upload_2.id + assert results[2].id == upload_1.id + + +def test_get_uploads_orders_by_processing_started_desc(sample_template): + days_ago = datetime.utcnow() - timedelta(days=3) + upload_1 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1), + created_at=days_ago, + job_status=JOB_STATUS_IN_PROGRESS) + upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=2), + created_at=days_ago, + job_status=JOB_STATUS_IN_PROGRESS) + + results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items + + assert len(results) == 2 + assert results[0].id == upload_1.id + assert results[1].id == upload_2.id + + +def test_get_uploads_orders_by_processing_started_and_created_at_desc(sample_template): + letter_template = create_precompiled_template(sample_template.service) + + days_ago = datetime.utcnow() - timedelta(days=4) + upload_0 = create_uploaded_letter(letter_template, service=letter_template.service) + upload_1 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1), + created_at=days_ago, + job_status=JOB_STATUS_IN_PROGRESS) + upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=2), + created_at=days_ago, + job_status=JOB_STATUS_IN_PROGRESS) + upload_3 = create_uploaded_letter(letter_template, service=letter_template.service, + created_at=datetime.utcnow() - timedelta(days=3)) + + results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items + + assert len(results) == 4 + assert results[0].id == upload_0.id + assert results[1].id == upload_1.id + assert results[2].id == upload_2.id + assert results[3].id == upload_3.id + + +def test_get_uploads_is_paginated(sample_template): + letter_template = create_precompiled_template(sample_template.service) + + upload_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(minutes=3)) + upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(minutes=2), + job_status=JOB_STATUS_IN_PROGRESS) + upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(minutes=1)) + upload_4 = create_job(sample_template, processing_started=datetime.utcnow(), job_status=JOB_STATUS_IN_PROGRESS) + + results = dao_get_uploads_by_service_id(sample_template.service_id, page=1, page_size=2) + + assert results.per_page == 2 + assert results.total == 4 + assert len(results.items) == 2 + assert results.items[0].id == upload_4.id + assert results.items[1].id == upload_3.id + + results = dao_get_uploads_by_service_id(sample_template.service_id, page=2, page_size=2) + + assert len(results.items) == 2 + assert results.items[0].id == upload_2.id + assert results.items[1].id == upload_1.id + + +def test_get_uploads_returns_empty_list(sample_service): + items = dao_get_uploads_by_service_id(sample_service.id).items + assert items == [] diff --git a/tests/app/upload/__init__.py b/tests/app/upload/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/app/upload/test_rest.py b/tests/app/upload/test_rest.py new file mode 100644 index 000000000..703236ee4 --- /dev/null +++ b/tests/app/upload/test_rest.py @@ -0,0 +1,180 @@ +from datetime import datetime, timedelta, date + +from freezegun import freeze_time + +from app.models import LETTER_TYPE, JOB_STATUS_FINISHED, JOB_STATUS_PENDING +from tests.app.db import create_job, create_notification, create_template, create_ft_notification_status +from tests.conftest import set_config + + +def create_uploaded_letter(letter_template, service, status='created', created_at=None): + return create_notification( + template=letter_template, + to_field="file-name", + status=status, + reference="dvla-reference", + client_reference="file-name", + one_off=True, + created_by_id=service.users[0].id, + created_at=created_at + ) + + +def create_precompiled_template(service): + return create_template( + service, + template_type=LETTER_TYPE, + template_name='Pre-compiled PDF', + subject='Pre-compiled PDF', + content="", + hidden=True, + postage="second", + ) + + +def test_get_uploads(admin_request, sample_template): + letter_template = create_precompiled_template(sample_template.service) + + upload_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(minutes=4)) + upload_2 = create_job(template=sample_template, + processing_started=datetime.utcnow() - timedelta(minutes=3), + job_status=JOB_STATUS_FINISHED) + upload_3 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(minutes=2)) + upload_4 = create_job(template=sample_template, + processing_started=datetime.utcnow() - timedelta(minutes=1), + job_status=JOB_STATUS_FINISHED) + upload_5 = create_job(template=sample_template, processing_started=None, + job_status=JOB_STATUS_PENDING, notification_count=10) + + service_id = sample_template.service.id + + resp_json = admin_request.get('upload.get_uploads_by_service', service_id=service_id) + data = resp_json['data'] + assert len(data) == 5 + assert data[0] == {'id': str(upload_5.id), + 'original_file_name': 'some.csv', + 'notification_count': 10, + 'created_at': upload_5.created_at.strftime("%Y-%m-%d %H:%M:%S"), + 'statistics': [], + 'upload_type': 'job'} + assert data[1] == {'id': str(upload_4.id), + 'original_file_name': 'some.csv', + 'notification_count': 1, + 'created_at': upload_4.created_at.strftime( + "%Y-%m-%d %H:%M:%S"), + 'statistics': [], + 'upload_type': 'job'} + assert data[2] == {'id': str(upload_3.id), + 'original_file_name': "file-name", + 'notification_count': 1, + 'created_at': upload_3.created_at.strftime("%Y-%m-%d %H:%M:%S"), + 'statistics': [{'count': 1, 'status': 'delivered'}], + 'upload_type': 'letter'} + assert data[3] == {'id': str(upload_2.id), + 'original_file_name': "some.csv", + 'notification_count': 1, + 'created_at': upload_2.created_at.strftime( + "%Y-%m-%d %H:%M:%S"), + 'statistics': [], + 'upload_type': 'job'} + assert data[4] == {'id': str(upload_1.id), + 'original_file_name': "file-name", + 'notification_count': 1, + 'created_at': upload_1.created_at.strftime("%Y-%m-%d %H:%M:%S"), + 'statistics': [{'count': 1, 'status': 'delivered'}], + 'upload_type': 'letter'} + + +def test_get_uploads_should_return_statistics(admin_request, sample_template): + now = datetime.utcnow() + earlier = datetime.utcnow() - timedelta(days=1) + job_1 = create_job(template=sample_template, job_status='pending') + job_2 = create_job(sample_template, processing_started=earlier) + for _ in range(0, 3): + create_notification(template=sample_template, job=job_2, status='created') + + job_3 = create_job(sample_template, processing_started=now) + for _ in range(0, 4): + create_notification(template=sample_template, job=job_3, status='sending') + letter_template = create_precompiled_template(sample_template.service) + letter_1 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(days=3)) + letter_2 = create_uploaded_letter(letter_template, sample_template.service, status='delivered', + created_at=datetime.utcnow() - timedelta(days=2)) + resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id)['data'] + assert len(resp_json) == 5 + assert resp_json[0]['id'] == str(job_1.id) + assert resp_json[0]['statistics'] == [] + assert resp_json[1]['id'] == str(job_3.id) + assert resp_json[1]['statistics'] == [{'status': 'sending', 'count': 4}] + assert resp_json[2]['id'] == str(job_2.id) + assert resp_json[2]['statistics'] == [{'status': 'created', 'count': 3}] + assert resp_json[3]['id'] == str(letter_2.id) + assert resp_json[3]['statistics'] == [{'status': 'delivered', 'count': 1}] + assert resp_json[4]['id'] == str(letter_1.id) + assert resp_json[4]['statistics'] == [{'status': 'delivered', 'count': 1}] + + +def test_get_uploads_should_paginate(admin_request, sample_template): + for _ in range(0, 10): + create_job(sample_template) + + with set_config(admin_request.app, 'PAGE_SIZE', 2): + resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id) + + assert len(resp_json['data']) == 2 + assert resp_json['page_size'] == 2 + assert resp_json['total'] == 10 + assert 'links' in resp_json + assert set(resp_json['links'].keys()) == {'next', 'last'} + + +def test_get_uploads_accepts_page_parameter(admin_request, sample_template): + for _ in range(0, 10): + create_job(sample_template) + + with set_config(admin_request.app, 'PAGE_SIZE', 2): + resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id, page=2) + + assert len(resp_json['data']) == 2 + assert resp_json['page_size'] == 2 + assert resp_json['total'] == 10 + assert 'links' in resp_json + assert set(resp_json['links'].keys()) == {'prev', 'next', 'last'} + + +@freeze_time('2017-06-10 12:00') +def test_get_uploads_should_retrieve_from_ft_notification_status_for_old_jobs(admin_request, sample_template): + # it's the 10th today, so 3 days should include all of 7th, 8th, 9th, and some of 10th. + just_three_days_ago = datetime(2017, 6, 6, 22, 59, 59) + not_quite_three_days_ago = just_three_days_ago + timedelta(seconds=1) + + job_1 = create_job(sample_template, created_at=just_three_days_ago, processing_started=just_three_days_ago) + job_2 = create_job(sample_template, created_at=just_three_days_ago, processing_started=not_quite_three_days_ago) + # is old but hasn't started yet (probably a scheduled job). We don't have any stats for this job yet. + job_3 = create_job(sample_template, created_at=just_three_days_ago, processing_started=None) + + # some notifications created more than three days ago, some created after the midnight cutoff + create_ft_notification_status(date(2017, 6, 6), job=job_1, notification_status='delivered', count=2) + create_ft_notification_status(date(2017, 6, 7), job=job_1, notification_status='delivered', count=4) + # job2's new enough + create_notification(job=job_2, status='created', created_at=not_quite_three_days_ago) + + # this isn't picked up because the job is too new + create_ft_notification_status(date(2017, 6, 7), job=job_2, notification_status='delivered', count=8) + # this isn't picked up - while the job is old, it started in last 3 days so we look at notification table instead + create_ft_notification_status(date(2017, 6, 7), job=job_3, notification_status='delivered', count=16) + + # this isn't picked up because we're using the ft status table for job_1 as it's old + create_notification(job=job_1, status='created', created_at=not_quite_three_days_ago) + + resp_json = admin_request.get('upload.get_uploads_by_service', service_id=sample_template.service_id)['data'] + + assert resp_json[0]['id'] == str(job_3.id) + assert resp_json[0]['statistics'] == [] + assert resp_json[1]['id'] == str(job_2.id) + assert resp_json[1]['statistics'] == [{'status': 'created', 'count': 1}] + assert resp_json[2]['id'] == str(job_1.id) + assert resp_json[2]['statistics'] == [{'status': 'delivered', 'count': 6}]