notifications-api/tests/app/celery/test_ftp_update_tasks.py

from collections import namedtuple, defaultdict
from datetime import datetime, date

import pytest
from freezegun import freeze_time
from flask import current_app

from app.exceptions import DVLAException, NotificationTechnicalFailureException
from app.models import (
    Job,
    Notification,
    NotificationHistory,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_SENDING,
    NOTIFICATION_TEMPORARY_FAILURE,
    NOTIFICATION_TECHNICAL_FAILURE,
)
from app.celery.tasks import (
    check_billable_units,
    get_billing_date_in_bst_from_filename,
    persist_daily_sorted_letter_counts,
    process_updates_from_file,
    update_dvla_job_to_error,
    update_letter_notifications_statuses,
    update_letter_notifications_to_error,
    update_letter_notifications_to_sent_to_dvla
)
from app.dao.daily_sorted_letter_dao import dao_get_daily_sorted_letter_by_billing_day

from tests.app.db import create_notification, create_service_callback_api
from tests.conftest import set_config


@pytest.fixture
def notification_update():
    """
    Returns a namedtuple to use as the argument for the check_billable_units function
    """
    NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
    return NotificationUpdate('REFERENCE_ABC', 'sent', '1', 'cost')


def test_update_dvla_job_to_error(sample_letter_template, sample_letter_job):
    create_notification(template=sample_letter_template, job=sample_letter_job)
    create_notification(template=sample_letter_template, job=sample_letter_job)
    update_dvla_job_to_error(job_id=sample_letter_job.id)

    updated_notifications = Notification.query.all()
    for n in updated_notifications:
        assert n.status == 'created'
        assert not n.sent_by

    assert Job.query.filter_by(id=sample_letter_job.id).one().job_status == 'error'


def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker):
    invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)

    with pytest.raises(DVLAException) as e:
        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
    assert 'DVLA response file: {} has an invalid format'.format('NOTIFY-20170823160812-RSP.TXT') in str(e)


def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history(
    sample_letter_template,
    mocker
):
    valid_file = 'ref-foo|Sent|1|Unsorted'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    notification = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                                       billable_units=1)
    Notification.query.filter_by(id=notification.id).delete()

    update_letter_notifications_statuses(filename="NOTIFY-20170823160812-RSP.TXT")

    updated_history = NotificationHistory.query.filter_by(id=notification.id).one()
    assert updated_history.status == NOTIFICATION_DELIVERED


def test_update_letter_notifications_statuses_raises_dvla_exception(notify_api, mocker, sample_letter_template):
    valid_file = 'ref-foo|Failed|1|Unsorted'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                        billable_units=0)

    with pytest.raises(DVLAException) as e:
        update_letter_notifications_statuses(filename="failed.txt")
    failed = ["ref-foo"]
    assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
        filename="failed.txt", failures=failed
    ) in str(e)


def test_update_letter_notifications_statuses_raises_error_for_unknown_sorted_status(
    notify_api,
    mocker,
    sample_letter_template
):
    sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
    sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
    valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Error'.format(
        sent_letter_1.reference, sent_letter_2.reference)

    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    with pytest.raises(DVLAException) as e:
        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')

    assert "DVLA response file: {filename} contains unknown Sorted status {unknown_status}".format(
        filename="NOTIFY-20170823160812-RSP.TXT", unknown_status="{'Error'}"
    ) in str(e)


def test_update_letter_notifications_statuses_still_raises_temp_failure_error_with_unknown_sorted_status(
    notify_api,
    mocker,
    sample_letter_template
):
    valid_file = 'ref-foo|Failed|1|unknown'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                        billable_units=0)

    with pytest.raises(DVLAException) as e:
        update_letter_notifications_statuses(filename="failed.txt")

    failed = ["ref-foo"]
    assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
        filename="failed.txt", failures=failed
    ) in str(e)


def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
    s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')

    with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
        s3_mock.assert_called_with('{}-ftp'.format(
            current_app.config['NOTIFY_EMAIL_DOMAIN']),
            'NOTIFY-20170823160812-RSP.TXT'
        )


def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')

    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')

    update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')


def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    updates = process_updates_from_file(valid_file)

    assert len(updates) == 2

    assert updates[0].reference == 'ref-foo'
    assert updates[0].status == 'Sent'
    assert updates[0].page_count == '1'
    assert updates[0].cost_threshold == 'Unsorted'

    assert updates[1].reference == 'ref-bar'
    assert updates[1].status == 'Sent'
    assert updates[1].page_count == '2'
    assert updates[1].cost_threshold == 'Sorted'


def test_update_letter_notifications_statuses_persisted(notify_api, mocker, sample_letter_template):
    sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                                      billable_units=0)
    failed_letter = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING,
                                        billable_units=0)
    create_service_callback_api(service=sample_letter_template.service, url="https://original_url.com")
    valid_file = '{}|Sent|1|Unsorted\n{}|Failed|2|Sorted'.format(
        sent_letter.reference, failed_letter.reference)
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    with pytest.raises(expected_exception=DVLAException) as e:
        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')

    assert sent_letter.status == NOTIFICATION_DELIVERED
    assert sent_letter.billable_units == 1
    assert sent_letter.updated_at
    assert failed_letter.status == NOTIFICATION_TEMPORARY_FAILURE
    assert failed_letter.billable_units == 2
    assert failed_letter.updated_at
    assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
        filename="NOTIFY-20170823160812-RSP.TXT", failures=[format(failed_letter.reference)]) in str(e)


def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count(
    notify_api,
    mocker,
    sample_letter_template
):
    sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
    sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
    valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Sorted'.format(
        sent_letter_1.reference, sent_letter_2.reference)

    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    persist_letter_count_mock = mocker.patch('app.celery.tasks.persist_daily_sorted_letter_counts')

    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')

    persist_letter_count_mock.assert_called_once_with(day=date(2017, 8, 23),
                                                      file_name='NOTIFY-20170823160812-RSP.TXT',
                                                      sorted_letter_counts={'Unsorted': 1, 'Sorted': 1})


def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count_with_no_sorted_values(
    notify_api,
    mocker,
    sample_letter_template,
    notify_db_session
):
    sent_letter_1 = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING)
    sent_letter_2 = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING)
    valid_file = '{}|Sent|1|Unsorted\n{}|Sent|2|Unsorted'.format(
        sent_letter_1.reference, sent_letter_2.reference)
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')

    daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))

    assert daily_sorted_letter.unsorted_count == 2
    assert daily_sorted_letter.sorted_count == 0


def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(notify_api, mocker,
                                                                                sample_letter_template):
    sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                                      billable_units=0)
    valid_file = '{}|Sent|1|Unsorted\n'.format(sent_letter.reference)
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    send_mock = mocker.patch(
        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
    )

    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
    send_mock.assert_not_called()


def test_update_letter_notifications_to_sent_to_dvla_updates_based_on_notification_references(
    client,
    sample_letter_template
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')

    dt = datetime.utcnow()
    with freeze_time(dt):
        update_letter_notifications_to_sent_to_dvla([first.reference])

    assert first.status == NOTIFICATION_SENDING
    assert first.sent_by == 'dvla'
    assert first.sent_at == dt
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED


def test_update_letter_notifications_to_error_updates_based_on_notification_references(
    client,
    sample_letter_template,
    mocker
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')
    create_service_callback_api(service=sample_letter_template.service, url="https://original_url.com")
    dt = datetime.utcnow()
    with freeze_time(dt):
        with pytest.raises(NotificationTechnicalFailureException) as e:
            update_letter_notifications_to_error([first.reference])
    assert first.reference in e.value.message

    assert first.status == NOTIFICATION_TECHNICAL_FAILURE
    assert first.sent_by is None
    assert first.sent_at is None
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED


def test_check_billable_units_when_billable_units_matches_page_count(
    client,
    sample_letter_template,
    mocker,
    notification_update
):
    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')

    create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=1)

    check_billable_units(notification_update)

    mock_logger.assert_not_called()


def test_check_billable_units_when_billable_units_does_not_match_page_count(
    client,
    sample_letter_template,
    mocker,
    notification_update
):
    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')

    notification = create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=3)

    check_billable_units(notification_update)

    mock_logger.assert_called_once_with(
        'Notification with id {} had 3 billable_units but a page count of 1'.format(notification.id)
    )


@pytest.mark.parametrize('filename_date, billing_date', [
    ('20170820230000', date(2017, 8, 21)),
    ('20170120230000', date(2017, 1, 20))
])
def test_get_billing_date_in_bst_from_filename(filename_date, billing_date):
    filename = 'NOTIFY-{}-RSP.TXT'.format(filename_date)
    result = get_billing_date_in_bst_from_filename(filename)

    assert result == billing_date


@freeze_time("2018-01-11 09:00:00")
def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(client, notify_db_session):
    letter_counts = defaultdict(int, **{'Unsorted': 5, 'Sorted': 1})
    persist_daily_sorted_letter_counts(date.today(), "test.txt", letter_counts)
    day = dao_get_daily_sorted_letter_by_billing_day(date.today())

    assert day.unsorted_count == 5
    assert day.sorted_count == 1