remove record-daily-sorted-count task

app/celery/tasks.py

@@ -446,26 +446,6 @@ def handle_exception(task, notification, notification_id, exc):
             current_app.logger.error('Max retry failed' + retry_msg)
 
 
-@notify_celery.task(bind=True, name="record-daily-sorted-counts")
-def record_daily_sorted_counts(self, filename):
-    sorted_letter_counts = defaultdict(int)
-    notification_updates = parse_dvla_file(filename)
-    for update in notification_updates:
-        sorted_letter_counts[update.cost_threshold.lower()] += 1
-
-    unknown_status = sorted_letter_counts.keys() - {'unsorted', 'sorted'}
-    if unknown_status:
-        message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
-            filename, unknown_status.__repr__()
-        )
-        raise DVLAException(message)
-
-    billing_date = get_local_billing_date_from_filename(filename)
-    persist_daily_sorted_letter_counts(day=billing_date,
-                                       file_name=filename,
-                                       sorted_letter_counts=sorted_letter_counts)
-
-
 def parse_dvla_file(filename):
     bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
     response_file_content = s3.get_s3_file(bucket_location, filename)

app/commands.py

@@ -19,7 +19,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app import db
 from app.aws import s3
-from app.celery.tasks import process_row, record_daily_sorted_counts
+from app.celery.tasks import process_row
 from app.config import QueueNames
 from app.dao.annual_billing_dao import (
     dao_create_or_update_annual_billing_for_year,
@@ -321,18 +321,6 @@ def update_jobs_archived_flag(start_date, end_date):
     current_app.logger.info('Total archived jobs = {}'.format(total_updated))
 
 
-@notify_command(name='replay-daily-sorted-count-files')
-@click.option('-f', '--file_extension', required=False, help="File extension to search for, defaults to rs.txt")
-@statsd(namespace="tasks")
-def replay_daily_sorted_count_files(file_extension):
-    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
-    for filename in s3.get_list_of_files_by_suffix(bucket_name=bucket_location,
-                                                   subfolder='root/dispatch',
-                                                   suffix=file_extension or '.rs.txt'):
-        print("Create task to record daily sorted counts for file: ", filename)
-        record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
-
-
 @notify_command(name='populate-organisations-from-file')
 @click.option('-f', '--file_name', required=True,
               help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains")

app/notifications/notifications_letter_callback.py

@@ -3,9 +3,6 @@ from functools import wraps
 
 from flask import Blueprint, current_app, jsonify, request
 
-from app.celery.tasks import (
-    record_daily_sorted_counts,
-)
 from app.config import QueueNames
 from app.notifications.utils import autoconfirm_subscription
 from app.schema_validation import validate
@@ -37,23 +34,3 @@ def validate_schema(schema):
             return f(*args, **kw)
         return wrapper
     return decorator
-
-
-@letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST'])
-@validate_schema(dvla_sns_callback_schema)
-def process_letter_response():
-    req_json = request.get_json(force=True)
-    current_app.logger.debug('Received SNS callback: {}'.format(req_json))
-    if not autoconfirm_subscription(req_json):
-        # The callback should have one record for an S3 Put Event.
-        message = json.loads(req_json['Message'])
-        filename = message['Records'][0]['s3']['object']['key']
-        current_app.logger.info('Received file from DVLA: {}'.format(filename))
-
-        if filename.lower().endswith('rs.txt') or filename.lower().endswith('rsp.txt'):
-            current_app.logger.info('DVLA callback: Calling task to update letter notifications')
-            record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
-
-    return jsonify(
-        result="success", message="DVLA callback succeeded"
-    ), 200

tests/app/celery/test_ftp_update_tasks.py

@@ -11,7 +11,6 @@ from app.celery.tasks import (
     get_local_billing_date_from_filename,
     persist_daily_sorted_letter_counts,
     process_updates_from_file,
-    record_daily_sorted_counts,
     update_letter_notifications_to_error,
     update_letter_notifications_to_sent_to_dvla,
 )
@@ -134,77 +133,3 @@ def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(cli
     assert day.unsorted_count == 5
     assert day.sorted_count == 1
 
-
-def test_record_daily_sorted_counts_persists_daily_sorted_letter_count(
-    notify_api,
-    notify_db_session,
-    mocker,
-):
-    valid_file = 'Letter1|Sent|1|uNsOrTeD\nLetter2|Sent|2|SORTED\nLetter3|Sent|2|Sorted'
-
-    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
-
-    assert DailySortedLetter.query.count() == 0
-
-    record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
-
-    daily_sorted_counts = DailySortedLetter.query.all()
-    assert len(daily_sorted_counts) == 1
-    assert daily_sorted_counts[0].sorted_count == 2
-    assert daily_sorted_counts[0].unsorted_count == 1
-
-
-def test_record_daily_sorted_counts_raises_dvla_exception_with_unknown_sorted_status(
-    notify_api,
-    mocker,
-):
-    file_contents = 'ref-foo|Failed|1|invalid\nrow_2|Failed|1|MM'
-    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=file_contents)
-    filename = "failed.txt"
-    with pytest.raises(DVLAException) as e:
-        record_daily_sorted_counts(filename=filename)
-
-    assert "DVLA response file: {} contains unknown Sorted status".format(filename) in e.value.message
-    assert "'mm'" in e.value.message
-    assert "'invalid'" in e.value.message
-
-
-def test_record_daily_sorted_counts_persists_daily_sorted_letter_count_with_no_sorted_values(
-    notify_api,
-    mocker,
-    notify_db_session
-):
-    valid_file = 'Letter1|Sent|1|Unsorted\nLetter2|Sent|2|Unsorted'
-    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
-
-    record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
-
-    daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
-
-    assert daily_sorted_letter.unsorted_count == 2
-    assert daily_sorted_letter.sorted_count == 0
-
-
-def test_record_daily_sorted_counts_can_run_twice_for_same_file(
-    notify_api,
-    mocker,
-    notify_db_session
-):
-    valid_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted'
-    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
-
-    record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
-
-    daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
-
-    assert daily_sorted_letter.unsorted_count == 1
-    assert daily_sorted_letter.sorted_count == 1
-
-    updated_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted\nLetter3|Sent|2|Unsorted'
-    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=updated_file)
-
-    record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
-    daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
-
-    assert daily_sorted_letter.unsorted_count == 2
-    assert daily_sorted_letter.sorted_count == 1