mirror of
https://github.com/GSA/notifications-api.git
synced 2026-05-22 01:31:36 -04:00
remove record-daily-sorted-count task
This commit is contained in:
stvnrlly
@@ -446,26 +446,6 @@ def handle_exception(task, notification, notification_id, exc):
|
||||
current_app.logger.error('Max retry failed' + retry_msg)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="record-daily-sorted-counts")
|
||||
def record_daily_sorted_counts(self, filename):
|
||||
sorted_letter_counts = defaultdict(int)
|
||||
notification_updates = parse_dvla_file(filename)
|
||||
for update in notification_updates:
|
||||
sorted_letter_counts[update.cost_threshold.lower()] += 1
|
||||
|
||||
unknown_status = sorted_letter_counts.keys() - {'unsorted', 'sorted'}
|
||||
if unknown_status:
|
||||
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
|
||||
filename, unknown_status.__repr__()
|
||||
)
|
||||
raise DVLAException(message)
|
||||
|
||||
billing_date = get_local_billing_date_from_filename(filename)
|
||||
persist_daily_sorted_letter_counts(day=billing_date,
|
||||
file_name=filename,
|
||||
sorted_letter_counts=sorted_letter_counts)
|
||||
|
||||
|
||||
def parse_dvla_file(filename):
|
||||
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
|
||||
response_file_content = s3.get_s3_file(bucket_location, filename)
|
||||
|
||||
@@ -19,7 +19,7 @@ from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from app import db
|
||||
from app.aws import s3
|
||||
from app.celery.tasks import process_row, record_daily_sorted_counts
|
||||
from app.celery.tasks import process_row
|
||||
from app.config import QueueNames
|
||||
from app.dao.annual_billing_dao import (
|
||||
dao_create_or_update_annual_billing_for_year,
|
||||
@@ -321,18 +321,6 @@ def update_jobs_archived_flag(start_date, end_date):
|
||||
current_app.logger.info('Total archived jobs = {}'.format(total_updated))
|
||||
|
||||
|
||||
@notify_command(name='replay-daily-sorted-count-files')
|
||||
@click.option('-f', '--file_extension', required=False, help="File extension to search for, defaults to rs.txt")
|
||||
@statsd(namespace="tasks")
|
||||
def replay_daily_sorted_count_files(file_extension):
|
||||
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
|
||||
for filename in s3.get_list_of_files_by_suffix(bucket_name=bucket_location,
|
||||
subfolder='root/dispatch',
|
||||
suffix=file_extension or '.rs.txt'):
|
||||
print("Create task to record daily sorted counts for file: ", filename)
|
||||
record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
|
||||
|
||||
|
||||
@notify_command(name='populate-organisations-from-file')
|
||||
@click.option('-f', '--file_name', required=True,
|
||||
help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains")
|
||||
|
||||
@@ -3,9 +3,6 @@ from functools import wraps
|
||||
|
||||
from flask import Blueprint, current_app, jsonify, request
|
||||
|
||||
from app.celery.tasks import (
|
||||
record_daily_sorted_counts,
|
||||
)
|
||||
from app.config import QueueNames
|
||||
from app.notifications.utils import autoconfirm_subscription
|
||||
from app.schema_validation import validate
|
||||
@@ -37,23 +34,3 @@ def validate_schema(schema):
|
||||
return f(*args, **kw)
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
@letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST'])
|
||||
@validate_schema(dvla_sns_callback_schema)
|
||||
def process_letter_response():
|
||||
req_json = request.get_json(force=True)
|
||||
current_app.logger.debug('Received SNS callback: {}'.format(req_json))
|
||||
if not autoconfirm_subscription(req_json):
|
||||
# The callback should have one record for an S3 Put Event.
|
||||
message = json.loads(req_json['Message'])
|
||||
filename = message['Records'][0]['s3']['object']['key']
|
||||
current_app.logger.info('Received file from DVLA: {}'.format(filename))
|
||||
|
||||
if filename.lower().endswith('rs.txt') or filename.lower().endswith('rsp.txt'):
|
||||
current_app.logger.info('DVLA callback: Calling task to update letter notifications')
|
||||
record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
|
||||
|
||||
return jsonify(
|
||||
result="success", message="DVLA callback succeeded"
|
||||
), 200
|
||||
|
||||
@@ -11,7 +11,6 @@ from app.celery.tasks import (
|
||||
get_local_billing_date_from_filename,
|
||||
persist_daily_sorted_letter_counts,
|
||||
process_updates_from_file,
|
||||
record_daily_sorted_counts,
|
||||
update_letter_notifications_to_error,
|
||||
update_letter_notifications_to_sent_to_dvla,
|
||||
)
|
||||
@@ -134,77 +133,3 @@ def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(cli
|
||||
assert day.unsorted_count == 5
|
||||
assert day.sorted_count == 1
|
||||
|
||||
|
||||
def test_record_daily_sorted_counts_persists_daily_sorted_letter_count(
|
||||
notify_api,
|
||||
notify_db_session,
|
||||
mocker,
|
||||
):
|
||||
valid_file = 'Letter1|Sent|1|uNsOrTeD\nLetter2|Sent|2|SORTED\nLetter3|Sent|2|Sorted'
|
||||
|
||||
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
|
||||
|
||||
assert DailySortedLetter.query.count() == 0
|
||||
|
||||
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
|
||||
|
||||
daily_sorted_counts = DailySortedLetter.query.all()
|
||||
assert len(daily_sorted_counts) == 1
|
||||
assert daily_sorted_counts[0].sorted_count == 2
|
||||
assert daily_sorted_counts[0].unsorted_count == 1
|
||||
|
||||
|
||||
def test_record_daily_sorted_counts_raises_dvla_exception_with_unknown_sorted_status(
|
||||
notify_api,
|
||||
mocker,
|
||||
):
|
||||
file_contents = 'ref-foo|Failed|1|invalid\nrow_2|Failed|1|MM'
|
||||
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=file_contents)
|
||||
filename = "failed.txt"
|
||||
with pytest.raises(DVLAException) as e:
|
||||
record_daily_sorted_counts(filename=filename)
|
||||
|
||||
assert "DVLA response file: {} contains unknown Sorted status".format(filename) in e.value.message
|
||||
assert "'mm'" in e.value.message
|
||||
assert "'invalid'" in e.value.message
|
||||
|
||||
|
||||
def test_record_daily_sorted_counts_persists_daily_sorted_letter_count_with_no_sorted_values(
|
||||
notify_api,
|
||||
mocker,
|
||||
notify_db_session
|
||||
):
|
||||
valid_file = 'Letter1|Sent|1|Unsorted\nLetter2|Sent|2|Unsorted'
|
||||
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
|
||||
|
||||
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
|
||||
|
||||
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
|
||||
|
||||
assert daily_sorted_letter.unsorted_count == 2
|
||||
assert daily_sorted_letter.sorted_count == 0
|
||||
|
||||
|
||||
def test_record_daily_sorted_counts_can_run_twice_for_same_file(
|
||||
notify_api,
|
||||
mocker,
|
||||
notify_db_session
|
||||
):
|
||||
valid_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted'
|
||||
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
|
||||
|
||||
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
|
||||
|
||||
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
|
||||
|
||||
assert daily_sorted_letter.unsorted_count == 1
|
||||
assert daily_sorted_letter.sorted_count == 1
|
||||
|
||||
updated_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted\nLetter3|Sent|2|Unsorted'
|
||||
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=updated_file)
|
||||
|
||||
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
|
||||
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
|
||||
|
||||
assert daily_sorted_letter.unsorted_count == 2
|
||||
assert daily_sorted_letter.sorted_count == 1
|
||||
|
||||
Reference in New Issue
Block a user