From 119c5800c4a20552d4aee531bcecb9ff79cd0537 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 7 Dec 2022 12:24:12 -0500 Subject: [PATCH] remove record-daily-sorted-count task --- app/celery/tasks.py | 20 ----- app/commands.py | 14 +--- .../notifications_letter_callback.py | 23 ------ tests/app/celery/test_ftp_update_tasks.py | 75 ------------------- 4 files changed, 1 insertion(+), 131 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index cafce5b2e..27b528e67 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -446,26 +446,6 @@ def handle_exception(task, notification, notification_id, exc): current_app.logger.error('Max retry failed' + retry_msg) -@notify_celery.task(bind=True, name="record-daily-sorted-counts") -def record_daily_sorted_counts(self, filename): - sorted_letter_counts = defaultdict(int) - notification_updates = parse_dvla_file(filename) - for update in notification_updates: - sorted_letter_counts[update.cost_threshold.lower()] += 1 - - unknown_status = sorted_letter_counts.keys() - {'unsorted', 'sorted'} - if unknown_status: - message = 'DVLA response file: {} contains unknown Sorted status {}'.format( - filename, unknown_status.__repr__() - ) - raise DVLAException(message) - - billing_date = get_local_billing_date_from_filename(filename) - persist_daily_sorted_letter_counts(day=billing_date, - file_name=filename, - sorted_letter_counts=sorted_letter_counts) - - def parse_dvla_file(filename): bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) response_file_content = s3.get_s3_file(bucket_location, filename) diff --git a/app/commands.py b/app/commands.py index c7a9b1dea..4817f3cdb 100644 --- a/app/commands.py +++ b/app/commands.py @@ -19,7 +19,7 @@ from sqlalchemy.orm.exc import NoResultFound from app import db from app.aws import s3 -from app.celery.tasks import process_row, record_daily_sorted_counts +from app.celery.tasks import process_row from app.config import QueueNames from app.dao.annual_billing_dao import ( dao_create_or_update_annual_billing_for_year, @@ -321,18 +321,6 @@ def update_jobs_archived_flag(start_date, end_date): current_app.logger.info('Total archived jobs = {}'.format(total_updated)) -@notify_command(name='replay-daily-sorted-count-files') -@click.option('-f', '--file_extension', required=False, help="File extension to search for, defaults to rs.txt") -@statsd(namespace="tasks") -def replay_daily_sorted_count_files(file_extension): - bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) - for filename in s3.get_list_of_files_by_suffix(bucket_name=bucket_location, - subfolder='root/dispatch', - suffix=file_extension or '.rs.txt'): - print("Create task to record daily sorted counts for file: ", filename) - record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY) - - @notify_command(name='populate-organisations-from-file') @click.option('-f', '--file_name', required=True, help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains") diff --git a/app/notifications/notifications_letter_callback.py b/app/notifications/notifications_letter_callback.py index 0e343cb0e..c970bc144 100644 --- a/app/notifications/notifications_letter_callback.py +++ b/app/notifications/notifications_letter_callback.py @@ -3,9 +3,6 @@ from functools import wraps from flask import Blueprint, current_app, jsonify, request -from app.celery.tasks import ( - record_daily_sorted_counts, -) from app.config import QueueNames from app.notifications.utils import autoconfirm_subscription from app.schema_validation import validate @@ -37,23 +34,3 @@ def validate_schema(schema): return f(*args, **kw) return wrapper return decorator - - -@letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST']) -@validate_schema(dvla_sns_callback_schema) -def process_letter_response(): - req_json = request.get_json(force=True) - current_app.logger.debug('Received SNS callback: {}'.format(req_json)) - if not autoconfirm_subscription(req_json): - # The callback should have one record for an S3 Put Event. - message = json.loads(req_json['Message']) - filename = message['Records'][0]['s3']['object']['key'] - current_app.logger.info('Received file from DVLA: {}'.format(filename)) - - if filename.lower().endswith('rs.txt') or filename.lower().endswith('rsp.txt'): - current_app.logger.info('DVLA callback: Calling task to update letter notifications') - record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY) - - return jsonify( - result="success", message="DVLA callback succeeded" - ), 200 diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py index 862c17aae..c4bad2e12 100644 --- a/tests/app/celery/test_ftp_update_tasks.py +++ b/tests/app/celery/test_ftp_update_tasks.py @@ -11,7 +11,6 @@ from app.celery.tasks import ( get_local_billing_date_from_filename, persist_daily_sorted_letter_counts, process_updates_from_file, - record_daily_sorted_counts, update_letter_notifications_to_error, update_letter_notifications_to_sent_to_dvla, ) @@ -134,77 +133,3 @@ def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(cli assert day.unsorted_count == 5 assert day.sorted_count == 1 - -def test_record_daily_sorted_counts_persists_daily_sorted_letter_count( - notify_api, - notify_db_session, - mocker, -): - valid_file = 'Letter1|Sent|1|uNsOrTeD\nLetter2|Sent|2|SORTED\nLetter3|Sent|2|Sorted' - - mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) - - assert DailySortedLetter.query.count() == 0 - - record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT') - - daily_sorted_counts = DailySortedLetter.query.all() - assert len(daily_sorted_counts) == 1 - assert daily_sorted_counts[0].sorted_count == 2 - assert daily_sorted_counts[0].unsorted_count == 1 - - -def test_record_daily_sorted_counts_raises_dvla_exception_with_unknown_sorted_status( - notify_api, - mocker, -): - file_contents = 'ref-foo|Failed|1|invalid\nrow_2|Failed|1|MM' - mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=file_contents) - filename = "failed.txt" - with pytest.raises(DVLAException) as e: - record_daily_sorted_counts(filename=filename) - - assert "DVLA response file: {} contains unknown Sorted status".format(filename) in e.value.message - assert "'mm'" in e.value.message - assert "'invalid'" in e.value.message - - -def test_record_daily_sorted_counts_persists_daily_sorted_letter_count_with_no_sorted_values( - notify_api, - mocker, - notify_db_session -): - valid_file = 'Letter1|Sent|1|Unsorted\nLetter2|Sent|2|Unsorted' - mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) - - record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT') - - daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23)) - - assert daily_sorted_letter.unsorted_count == 2 - assert daily_sorted_letter.sorted_count == 0 - - -def test_record_daily_sorted_counts_can_run_twice_for_same_file( - notify_api, - mocker, - notify_db_session -): - valid_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted' - mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) - - record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT') - - daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23)) - - assert daily_sorted_letter.unsorted_count == 1 - assert daily_sorted_letter.sorted_count == 1 - - updated_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted\nLetter3|Sent|2|Unsorted' - mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=updated_file) - - record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT') - daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23)) - - assert daily_sorted_letter.unsorted_count == 2 - assert daily_sorted_letter.sorted_count == 1