carve out ftp

app/config.py

@@ -19,7 +19,6 @@ class QueueNames(object):
     JOBS = 'job-tasks'
     RETRY = 'retry-tasks'
     NOTIFY = 'notify-internal-tasks'
-    PROCESS_FTP = 'process-ftp-tasks'
     CALLBACKS = 'service-callbacks'
     CALLBACKS_RETRY = 'service-callbacks-retry'
     SMS_CALLBACKS = 'sms-callbacks'

app/dao/notifications_dao.py

@@ -569,35 +569,6 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
-    """
-    Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
-    to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
-    get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese  start sending off
-    tasks to the notify-ftp app to send them.
-
-    CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
-    there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
-    and we should avoid modifying state of returned objects.
-
-    For more reading:
-    https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
-    https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
-    """
-    notifications = Notification.query.filter(
-        Notification.created_at < convert_local_timezone_to_utc(print_run_deadline),
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.postage == postage,
-        Notification.billable_units > 0
-    ).order_by(
-        Notification.service_id,
-        Notification.created_at
-    ).yield_per(query_limit)
-    return notifications
-
-
 def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
     notifications = db.session.query(
         func.count(Notification.id).label('letters_count'),

docs/queues-and-tasks.md

@@ -12,7 +12,6 @@ There are a bunch of queues:
 - job tasks
 - retry tasks
 - notify internal tasks
-- process ftp tasks
 - service callbacks
 - service callbacks retry
 - letter tasks

tests/app/dao/notification_dao/test_notification_dao.py

@@ -12,7 +12,6 @@ from app.dao.notifications_dao import (
     dao_delete_notifications_by_id,
     dao_get_last_notification_added_for_job_id,
     dao_get_letters_and_sheets_volume_by_postage,
-    dao_get_letters_to_be_printed,
     dao_get_notification_by_reference,
     dao_get_notification_count_for_job_id,
     dao_get_notification_history_by_reference,
@@ -1563,44 +1562,6 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_
     assert len(results) == 0
 
 
-def test_letters_to_be_printed_sort_by_service(notify_db_session):
-    first_service = create_service(service_name='first service', service_id='3a5cea08-29fd-4bb9-b582-8dedd928b149')
-    second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704')
-    first_template = create_template(service=first_service, template_type='letter', postage='second')
-    second_template = create_template(service=second_service, template_type='letter', postage='second')
-    letters_ordered_by_service_then_time = [
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30)),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30)),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30)),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30)),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34))
-    ]
-
-    results = list(
-        dao_get_letters_to_be_printed(print_run_deadline=datetime(2020, 12, 1, 17, 30), postage='second', query_limit=4)
-    )
-    assert [x.id for x in results] == [x.id for x in letters_ordered_by_service_then_time]
-
-
-def test_letters_to_be_printed_does_not_include_letters_without_billable_units_set(
-        notify_db_session, sample_letter_template):
-    included_letter = create_notification(
-        template=sample_letter_template, created_at=datetime(2020, 12, 1, 9, 30), billable_units=3)
-    create_notification(
-        template=sample_letter_template, created_at=datetime(2020, 12, 1, 9, 31), billable_units=0)
-
-    results = list(
-        dao_get_letters_to_be_printed(print_run_deadline=datetime(2020, 12, 1, 17, 30), postage='second', query_limit=4)
-    )
-    assert len(results) == 1
-    assert results[0].id == included_letter.id
-
-
 def test_dao_get_letters_and_sheets_volume_by_postage(notify_db_session):
     first_service = create_service(service_name='first service', service_id='3a5cea08-29fd-4bb9-b582-8dedd928b149')
     second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704')