darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-02 17:48:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

reformat

Browse Source
This commit is contained in:
Kenneth Kehl
2023-08-29 14:54:30 -07:00
parent 19dcd7a48b
commit 1ecb747c6d
588 changed files with 34100 additions and 23589 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
app/celery/scheduled_tasks.py
View File

@@ -49,7 +49,9 @@ def run_scheduled_jobs():
try:
for job in dao_set_scheduled_jobs_to_pending():
process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
current_app.logger.info("Job ID {} added to process job queue".format(job.id))
current_app.logger.info(
"Job ID {} added to process job queue".format(job.id)
)
except SQLAlchemyError:
current_app.logger.exception("Failed to run scheduled jobs")
raise
@@ -61,7 +63,9 @@ def delete_verify_codes():
start = datetime.utcnow()
deleted = delete_codes_older_created_more_than_a_day_ago()
current_app.logger.info(
"Delete job started {} finished {} deleted {} verify codes".format(start, datetime.utcnow(), deleted)
"Delete job started {} finished {} deleted {} verify codes".format(
start, datetime.utcnow(), deleted
)
)
except SQLAlchemyError:
current_app.logger.exception("Failed to delete verify codes")
@@ -75,14 +79,16 @@ def delete_invitations():
deleted_invites = delete_invitations_created_more_than_two_days_ago()
deleted_invites += delete_org_invitations_created_more_than_two_days_ago()
current_app.logger.info(
"Delete job started {} finished {} deleted {} invitations".format(start, datetime.utcnow(), deleted_invites)
"Delete job started {} finished {} deleted {} invitations".format(
start, datetime.utcnow(), deleted_invites
)
)
except SQLAlchemyError:
current_app.logger.exception("Failed to delete invitations")
raise
@notify_celery.task(name='check-job-status')
@notify_celery.task(name="check-job-status")
def check_job_status():
"""
every x minutes do this check
@@ -101,19 +107,19 @@ def check_job_status():
incomplete_in_progress_jobs = Job.query.filter(
Job.job_status == JOB_STATUS_IN_PROGRESS,
between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago),
)
incomplete_pending_jobs = Job.query.filter(
Job.job_status == JOB_STATUS_PENDING,
Job.scheduled_for.isnot(None),
between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago),
)
jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
incomplete_pending_jobs
).order_by(
Job.processing_started, Job.scheduled_for
).all()
jobs_not_complete_after_30_minutes = (
incomplete_in_progress_jobs.union(incomplete_pending_jobs)
.order_by(Job.processing_started, Job.scheduled_for)
.all()
)
# temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
# if they haven't been re-processed in time.
@@ -125,52 +131,65 @@ def check_job_status():
if job_ids:
current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
process_incomplete_jobs.apply_async(
[job_ids],
queue=QueueNames.JOBS
)
process_incomplete_jobs.apply_async([job_ids], queue=QueueNames.JOBS)
@notify_celery.task(name='replay-created-notifications')
@notify_celery.task(name="replay-created-notifications")
def replay_created_notifications():
# if the notification has not be send after 1 hour, then try to resend.
resend_created_notifications_older_than = (60 * 60)
resend_created_notifications_older_than = 60 * 60
for notification_type in (EMAIL_TYPE, SMS_TYPE):
notifications_to_resend = notifications_not_yet_sent(
resend_created_notifications_older_than,
notification_type
resend_created_notifications_older_than, notification_type
)
if len(notifications_to_resend) > 0:
current_app.logger.info("Sending {} {} notifications "
"to the delivery queue because the notification "
"status was created.".format(len(notifications_to_resend), notification_type))
current_app.logger.info(
"Sending {} {} notifications "
"to the delivery queue because the notification "
"status was created.".format(
len(notifications_to_resend), notification_type
)
)
for n in notifications_to_resend:
send_notification_to_queue(notification=n)
@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
@notify_celery.task(name="check-for-missing-rows-in-completed-jobs")
def check_for_missing_rows_in_completed_jobs():
jobs = find_jobs_with_missing_rows()
for job in jobs:
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
(
recipient_csv,
template,
sender_id,
) = get_recipient_csv_and_template_and_sender_id(job)
missing_rows = find_missing_row_for_job(job.id, job.notification_count)
for row_to_process in missing_rows:
row = recipient_csv[row_to_process.missing_row]
current_app.logger.info(
"Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
"Processing missing row: {} for job: {}".format(
row_to_process.missing_row, job.id
)
)
process_row(row, template, job, job.service, sender_id=sender_id)
@notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
@notify_celery.task(
name="check-for-services-with-high-failure-rates-or-sending-to-tv-numbers"
)
def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
start_date = (datetime.utcnow() - timedelta(days=1))
start_date = datetime.utcnow() - timedelta(days=1)
end_date = datetime.utcnow()
message = ""
services_with_failures = dao_find_services_with_high_failure_rates(start_date=start_date, end_date=end_date)
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(start_date=start_date, end_date=end_date)
services_with_failures = dao_find_services_with_high_failure_rates(
start_date=start_date, end_date=end_date
)
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(
start_date=start_date, end_date=end_date
)
if services_with_failures:
message += "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours:\n".format(
@@ -178,17 +197,19 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
)
for service in services_with_failures:
service_dashboard = "{}/services/{}".format(
current_app.config['ADMIN_BASE_URL'],
current_app.config["ADMIN_BASE_URL"],
str(service.service_id),
)
message += "service: {} failure rate: {},\n".format(service_dashboard, service.permanent_failure_rate)
message += "service: {} failure rate: {},\n".format(
service_dashboard, service.permanent_failure_rate
)
elif services_sending_to_tv_numbers:
message += "{} service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n".format(
len(services_sending_to_tv_numbers)
)
for service in services_sending_to_tv_numbers:
service_dashboard = "{}/services/{}".format(
current_app.config['ADMIN_BASE_URL'],
current_app.config["ADMIN_BASE_URL"],
str(service.service_id),
)
message += "service: {} count of sms to tv numbers: {},\n".format(
@@ -198,13 +219,15 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
if services_with_failures or services_sending_to_tv_numbers:
current_app.logger.warning(message)
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
message += ("\nYou can find instructions for this ticket in our manual:\n"
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers") # noqa
if current_app.config["NOTIFY_ENVIRONMENT"] in ["live", "production", "test"]:
message += (
"\nYou can find instructions for this ticket in our manual:\n"
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers"
) # noqa
ticket = NotifySupportTicket(
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
message=message,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True
technical_ticket=True,
)
zendesk_client.send_ticket_to_zendesk(ticket)