darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-03 09:51:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove letters-related code (#175)

Browse Source 
This deletes a big ol' chunk of code related to letters. It's not everything—there are still a few things that might be tied to sms/email—but it's the the heart of letters function. SMS and email function should be untouched by this.

Areas affected:

- Things obviously about letters
- PDF tasks, used for precompiling letters
- Virus scanning, used for those PDFs
- FTP, used to send letters to the printer
- Postage stuff
This commit is contained in:
Steven Reilly
2023-03-02 20:20:31 -05:00
committed by GitHub
parent b07b95f795
commit ff4190a8eb
141 changed files with 1108 additions and 12083 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
app/celery/nightly_tasks.py
View File

@@ -1,15 +1,10 @@
from datetime import datetime, timedelta
import pytz
from flask import current_app
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
from notifications_utils.timezones import convert_utc_to_local_timezone
from sqlalchemy import func
from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery, statsd_client, zendesk_client
from app import notify_celery, statsd_client
from app.aws import s3
from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
from app.config import QueueNames
@@ -29,15 +24,7 @@ from app.dao.notifications_dao import (
from app.dao.service_data_retention_dao import (
fetch_service_data_retention_for_all_services_by_notification_type,
)
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
LETTER_TYPE,
NOTIFICATION_SENDING,
SMS_TYPE,
FactProcessingTime,
Notification,
)
from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
from app.utils import get_local_midnight_in_utc
@@ -47,12 +34,6 @@ def remove_sms_email_csv_files():
_remove_csv_files([EMAIL_TYPE, SMS_TYPE])
@notify_celery.task(name="remove_letter_jobs")
@cronitor("remove_letter_jobs")
def remove_letter_csv_files():
_remove_csv_files([LETTER_TYPE])
def _remove_csv_files(job_types):
jobs = dao_get_jobs_older_than_data_retention(notification_types=job_types)
for job in jobs:
@@ -65,7 +46,6 @@ def _remove_csv_files(job_types):
def delete_notifications_older_than_retention():
delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
@notify_celery.task(name="delete-sms-notifications")
@@ -80,12 +60,6 @@ def delete_email_notifications_older_than_retention():
_delete_notifications_older_than_retention_by_type('email')
@notify_celery.task(name="delete-letter-notifications")
@cronitor("delete-letter-notifications")
def delete_letter_notifications_older_than_retention():
_delete_notifications_older_than_retention_by_type('letter')
def _delete_notifications_older_than_retention_by_type(notification_type):
flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
@@ -185,110 +159,6 @@ def delete_inbound_sms():
raise
@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
@cronitor("raise-alert-if-letter-notifications-still-sending")
def raise_alert_if_letter_notifications_still_sending():
still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
if still_sending_count:
message = "There are {} letters in the 'sending' state from {}".format(
still_sending_count,
sent_date.strftime('%A %d %B')
)
# Only send alerts in production
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending" # noqa
ticket = NotifySupportTicket(
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
message=message,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
else:
current_app.logger.info(message)
def get_letter_notifications_still_sending_when_they_shouldnt_be():
today = datetime.utcnow().date()
# Do nothing on the weekend
if today.isoweekday() in {6, 7}: # sat, sun
return 0, None
if today.isoweekday() in {1, 2}: # mon, tues. look for files from before the weekend
offset_days = 4
else:
offset_days = 2
expected_sent_date = today - timedelta(days=offset_days)
q = Notification.query.filter(
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_SENDING,
Notification.key_type == KEY_TYPE_NORMAL,
func.date(Notification.sent_at) <= expected_sent_date
)
return q.count(), expected_sent_date
@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
@cronitor('raise-alert-if-no-letter-ack-file')
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
today_str = datetime.utcnow().strftime('%Y-%m-%d')
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=today_str + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().replace('.ZIP.TXT', ''))
# get acknowledgement file
ack_file_set = set()
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', '')) # noqa
message = '\n'.join([
"Letter ack file does not contain all zip files sent."
"",
f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n", # noqa
f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent", # noqa
f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
"",
f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
])
# strip empty element before comparison
ack_file_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_file_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
ticket = NotifySupportTicket(
subject="Letter acknowledge error",
message=message,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
current_app.logger.error(message)
if len(ack_file_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
)
@notify_celery.task(name='save-daily-notification-processing-time')
@cronitor("save-daily-notification-processing-time")
def save_daily_notification_processing_time(local_date=None):