Reduce extra S3 ops when working with letter PDFs

Previously we did some unnecessary work:

- Collate task. This had one S3 request to get a summary of the object,
which was then used in another request to get the full object. We only
need the size of the object, which is included in the summary [1].

- Archive task. This had one S3 request to get a summary of the object,
which was then used to make another request to delete it. We still need
both requests, but we can remove the S3.Object in the middle.

[1]: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#objectsummary
This commit is contained in:
Ben Thorner
2021-03-16 11:57:33 +00:00
parent ff7eebc90a
commit c76e789f1e
5 changed files with 22 additions and 38 deletions

View File

@@ -22,12 +22,11 @@ from sqlalchemy.sql.expression import case
from werkzeug.datastructures import MultiDict
from app import create_uuid, db
from app.aws.s3 import remove_s3_object
from app.clients.sms.firetext import (
get_message_status_and_reason_from_firetext_code,
)
from app.dao.dao_utils import transactional
from app.letters.utils import LetterPDFNotFound, find_letter_pdf_filename
from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
@@ -440,7 +439,6 @@ def _move_notifications_to_notification_history(notification_type, service_id, d
def _delete_letters_from_s3(
notification_type, service_id, date_to_delete_from, query_limit
):
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
letters_to_delete_from_s3 = db.session.query(
Notification
).filter(
@@ -454,8 +452,8 @@ def _delete_letters_from_s3(
).limit(query_limit).all()
for letter in letters_to_delete_from_s3:
try:
filename = find_letter_pdf_filename(letter)
remove_s3_object(bucket_name, filename)
letter_pdf = find_letter_pdf_in_s3(letter)
letter_pdf.delete()
except (ClientError, LetterPDFNotFound):
current_app.logger.exception(
"Could not delete S3 object for letter: {}".format(letter.id))