mirror of
https://github.com/GSA/notifications-api.git
synced 2026-07-19 14:04:20 -04:00
remove pdf collation task
This commit is contained in:
stvnrlly
@@ -126,168 +126,6 @@ def update_validation_failed_for_templated_letter(self, notification_id, page_co
|
||||
current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}")
|
||||
|
||||
|
||||
@notify_celery.task(name='collate-letter-pdfs-to-be-sent')
|
||||
@cronitor("collate-letter-pdfs-to-be-sent")
|
||||
def collate_letter_pdfs_to_be_sent():
|
||||
"""
|
||||
Finds all letters which are still waiting to be sent to DVLA for printing
|
||||
|
||||
This would usually be run at 5.50pm and collect up letters created between before 5:30pm today
|
||||
that have not yet been sent.
|
||||
If run after midnight, it will collect up letters created before 5:30pm the day before.
|
||||
"""
|
||||
print_run_date = convert_utc_to_local_timezone(datetime.utcnow())
|
||||
if print_run_date.time() < LETTER_PROCESSING_DEADLINE:
|
||||
print_run_date = print_run_date - timedelta(days=1)
|
||||
|
||||
print_run_deadline = print_run_date.replace(
|
||||
hour=17, minute=30, second=0, microsecond=0
|
||||
)
|
||||
_get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)
|
||||
|
||||
for postage in POSTAGE_TYPES:
|
||||
current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
|
||||
letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
|
||||
|
||||
for i, letters in enumerate(group_letters(letters_to_print)):
|
||||
filenames = [letter['Key'] for letter in letters]
|
||||
|
||||
service_id = letters[0]['ServiceId']
|
||||
organisation_id = letters[0]['OrganisationId']
|
||||
|
||||
hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
|
||||
# eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
|
||||
dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format(
|
||||
date=print_run_deadline.strftime("%Y-%m-%d"),
|
||||
postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
|
||||
num=i + 1,
|
||||
hash=hash,
|
||||
service_id=service_id,
|
||||
organisation_id=organisation_id
|
||||
)
|
||||
|
||||
current_app.logger.info(
|
||||
'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
|
||||
len(filenames),
|
||||
dvla_filename,
|
||||
sum(letter['Size'] for letter in letters)
|
||||
)
|
||||
)
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
|
||||
kwargs={
|
||||
'filenames_to_zip': filenames,
|
||||
'upload_filename': dvla_filename
|
||||
},
|
||||
queue=QueueNames.PROCESS_FTP,
|
||||
compression='zlib'
|
||||
)
|
||||
current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}")
|
||||
|
||||
current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
|
||||
|
||||
|
||||
def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
|
||||
letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
|
||||
send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date())
|
||||
|
||||
|
||||
def send_letters_volume_email_to_dvla(letters_volumes, date):
|
||||
personalisation = {
|
||||
'total_volume': 0,
|
||||
'first_class_volume': 0,
|
||||
'second_class_volume': 0,
|
||||
'international_volume': 0,
|
||||
'total_sheets': 0,
|
||||
'first_class_sheets': 0,
|
||||
"second_class_sheets": 0,
|
||||
'international_sheets': 0,
|
||||
'date': date.strftime("%d %B %Y")
|
||||
}
|
||||
for item in letters_volumes:
|
||||
personalisation['total_volume'] += item.letters_count
|
||||
personalisation['total_sheets'] += item.sheets_count
|
||||
if f"{item.postage}_class_volume" in personalisation:
|
||||
personalisation[f"{item.postage}_class_volume"] = item.letters_count
|
||||
personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
|
||||
else:
|
||||
personalisation["international_volume"] += item.letters_count
|
||||
personalisation["international_sheets"] += item.sheets_count
|
||||
|
||||
template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
|
||||
recipients = current_app.config['DVLA_EMAIL_ADDRESSES']
|
||||
reply_to = template.service.get_default_reply_to_email_address()
|
||||
service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
|
||||
|
||||
# avoid circular imports:
|
||||
from app.notifications.process_notifications import (
|
||||
persist_notification,
|
||||
send_notification_to_queue,
|
||||
)
|
||||
for recipient in recipients:
|
||||
saved_notification = persist_notification(
|
||||
template_id=template.id,
|
||||
template_version=template.version,
|
||||
recipient=recipient,
|
||||
service=service,
|
||||
personalisation=personalisation,
|
||||
notification_type=template.template_type,
|
||||
api_key_id=None,
|
||||
key_type=KEY_TYPE_NORMAL,
|
||||
reply_to_text=reply_to
|
||||
)
|
||||
|
||||
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
|
||||
|
||||
|
||||
def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
|
||||
letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
|
||||
for letter in letters_awaiting_sending:
|
||||
try:
|
||||
letter_pdf = find_letter_pdf_in_s3(letter)
|
||||
yield {
|
||||
"Key": letter_pdf.key,
|
||||
"Size": letter_pdf.size,
|
||||
"ServiceId": str(letter.service_id),
|
||||
"OrganisationId": str(letter.service.organisation_id)
|
||||
}
|
||||
except (BotoClientError, LetterPDFNotFound) as e:
|
||||
current_app.logger.exception(
|
||||
f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
|
||||
|
||||
|
||||
def group_letters(letter_pdfs):
|
||||
"""
|
||||
Group letters in chunks of MAX_LETTER_PDF_ZIP_FILESIZE. Will add files to lists, never going over that size.
|
||||
If a single file is (somehow) larger than MAX_LETTER_PDF_ZIP_FILESIZE that'll be in a list on it's own.
|
||||
If there are no files, will just exit (rather than yielding an empty list).
|
||||
"""
|
||||
running_filesize = 0
|
||||
list_of_files = []
|
||||
service_id = None
|
||||
for letter in letter_pdfs:
|
||||
if letter['Key'].lower().endswith('.pdf'):
|
||||
if not service_id:
|
||||
service_id = letter['ServiceId']
|
||||
if (
|
||||
running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']
|
||||
or len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
|
||||
or letter['ServiceId'] != service_id
|
||||
):
|
||||
yield list_of_files
|
||||
running_filesize = 0
|
||||
list_of_files = []
|
||||
service_id = None
|
||||
|
||||
if not service_id:
|
||||
service_id = letter['ServiceId']
|
||||
running_filesize += letter['Size']
|
||||
list_of_files.append(letter)
|
||||
|
||||
if list_of_files:
|
||||
yield list_of_files
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300)
|
||||
def sanitise_letter(self, filename):
|
||||
try:
|
||||
|
||||
@@ -300,11 +300,6 @@ class Config(object):
|
||||
},
|
||||
# The collate-letter-pdf does assume it is called in an hour that BST does not make a
|
||||
# difference to the truncate date which translates to the filename to process
|
||||
'collate-letter-pdfs-to-be-sent': {
|
||||
'task': 'collate-letter-pdfs-to-be-sent',
|
||||
'schedule': crontab(hour=17, minute=50),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'raise-alert-if-no-letter-ack-file': {
|
||||
'task': 'raise-alert-if-no-letter-ack-file',
|
||||
'schedule': crontab(hour=23, minute=00),
|
||||
|
||||
Reference in New Issue
Block a user