diff --git a/app/config.py b/app/config.py index 0e1da712a..7b3978154 100644 --- a/app/config.py +++ b/app/config.py @@ -19,7 +19,6 @@ class QueueNames(object): JOBS = 'job-tasks' RETRY = 'retry-tasks' NOTIFY = 'notify-internal-tasks' - PROCESS_FTP = 'process-ftp-tasks' CALLBACKS = 'service-callbacks' CALLBACKS_RETRY = 'service-callbacks-retry' SMS_CALLBACKS = 'sms-callbacks' diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 0ccb6e03b..f04ba73ce 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -569,35 +569,6 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ return notifications -def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000): - """ - Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k - to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the - get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese start sending off - tasks to the notify-ftp app to send them. - - CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then - there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique, - and we should avoid modifying state of returned objects. - - For more reading: - https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per - https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html - """ - notifications = Notification.query.filter( - Notification.created_at < convert_local_timezone_to_utc(print_run_deadline), - Notification.notification_type == LETTER_TYPE, - Notification.status == NOTIFICATION_CREATED, - Notification.key_type == KEY_TYPE_NORMAL, - Notification.postage == postage, - Notification.billable_units > 0 - ).order_by( - Notification.service_id, - Notification.created_at - ).yield_per(query_limit) - return notifications - - def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline): notifications = db.session.query( func.count(Notification.id).label('letters_count'), diff --git a/docs/queues-and-tasks.md b/docs/queues-and-tasks.md index c667b7832..2be48ad95 100644 --- a/docs/queues-and-tasks.md +++ b/docs/queues-and-tasks.md @@ -12,7 +12,6 @@ There are a bunch of queues: - job tasks - retry tasks - notify internal tasks -- process ftp tasks - service callbacks - service callbacks retry - letter tasks diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 8ae304ebf..4a6a1324f 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -12,7 +12,6 @@ from app.dao.notifications_dao import ( dao_delete_notifications_by_id, dao_get_last_notification_added_for_job_id, dao_get_letters_and_sheets_volume_by_postage, - dao_get_letters_to_be_printed, dao_get_notification_by_reference, dao_get_notification_count_for_job_id, dao_get_notification_history_by_reference, @@ -1563,44 +1562,6 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_ assert len(results) == 0 -def test_letters_to_be_printed_sort_by_service(notify_db_session): - first_service = create_service(service_name='first service', service_id='3a5cea08-29fd-4bb9-b582-8dedd928b149') - second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704') - first_template = create_template(service=first_service, template_type='letter', postage='second') - second_template = create_template(service=second_service, template_type='letter', postage='second') - letters_ordered_by_service_then_time = [ - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30)), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30)), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30)), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30)), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34)) - ] - - results = list( - dao_get_letters_to_be_printed(print_run_deadline=datetime(2020, 12, 1, 17, 30), postage='second', query_limit=4) - ) - assert [x.id for x in results] == [x.id for x in letters_ordered_by_service_then_time] - - -def test_letters_to_be_printed_does_not_include_letters_without_billable_units_set( - notify_db_session, sample_letter_template): - included_letter = create_notification( - template=sample_letter_template, created_at=datetime(2020, 12, 1, 9, 30), billable_units=3) - create_notification( - template=sample_letter_template, created_at=datetime(2020, 12, 1, 9, 31), billable_units=0) - - results = list( - dao_get_letters_to_be_printed(print_run_deadline=datetime(2020, 12, 1, 17, 30), postage='second', query_limit=4) - ) - assert len(results) == 1 - assert results[0].id == included_letter.id - - def test_dao_get_letters_and_sheets_volume_by_postage(notify_db_session): first_service = create_service(service_name='first service', service_id='3a5cea08-29fd-4bb9-b582-8dedd928b149') second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704')