darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-19 05:58:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

This massive set of changes uses the new queue names object throughout the app and tests.

Browse Source 
Lots of changes, all changing the line of code that puts things into queues, and the code that tests that.
This commit is contained in:
Martyn Inglis
2017-05-25 10:51:49 +01:00
parent 21586c917c
commit 2591d3a1df
28 changed files with 128 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/celery/provider_tasks.py
View File

@@ -3,6 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
from sqlalchemy.orm.exc import NoResultFound
from app import notify_celery
from app.config import QueueNames
from app.dao import notifications_dao
from app.dao.notifications_dao import update_notification_status_by_id
from app.statsd_decorators import statsd
@@ -46,7 +47,7 @@ def deliver_sms(self, notification_id):
current_app.logger.exception(
"SMS notification delivery for id: {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.exception(
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
@@ -70,7 +71,7 @@ def deliver_email(self, notification_id):
current_app.logger.exception(
"RETRY: Email notification {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id)

3
app/celery/scheduled_tasks.py
View File

@@ -23,6 +23,7 @@ from app.dao.provider_details_dao import (
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
from app.statsd_decorators import statsd
from app.celery.tasks import process_job
from app.config import QueueNames
@notify_celery.task(name="remove_csv_files")
@@ -39,7 +40,7 @@ def remove_csv_files():
def run_scheduled_jobs():
try:
for job in dao_set_scheduled_jobs_to_pending():
process_job.apply_async([str(job.id)], queue="process-job")
process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
current_app.logger.info("Job ID {} added to process job queue".format(job.id))
except SQLAlchemyError as e:
current_app.logger.exception("Failed to run scheduled jobs")

12
app/celery/statistics_tasks.py
View File

@@ -3,7 +3,6 @@ from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery
from flask import current_app
from app.models import JobStatistics
from app.statsd_decorators import statsd
from app.dao.statistics_dao import (
create_or_update_job_sending_statistics,
@@ -11,16 +10,17 @@ from app.dao.statistics_dao import (
)
from app.dao.notifications_dao import get_notification_by_id
from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
from app.config import QueueNames
def create_initial_notification_statistic_tasks(notification):
if notification.job_id and notification.status:
record_initial_job_statistics.apply_async((str(notification.id),), queue="statistics")
record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
def create_outcome_notification_statistic_tasks(notification):
if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
record_outcome_job_statistics.apply_async((str(notification.id),), queue="statistics")
record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
@notify_celery.task(bind=True, name='record_initial_job_statistics', max_retries=20, default_retry_delay=10)
@@ -35,7 +35,7 @@ def record_initial_job_statistics(self, notification_id):
raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
except SQLAlchemyError as e:
current_app.logger.exception(e)
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task record_initial_job_statistics failed for notification {}".format(
@@ -53,12 +53,12 @@ def record_outcome_job_statistics(self, notification_id):
if notification:
updated_count = update_job_stats_outcome_count(notification)
if updated_count == 0:
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
else:
raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
except SQLAlchemyError as e:
current_app.logger.exception(e)
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task update_job_stats_outcome_count failed for notification {}".format(

20
app/celery/tasks.py
View File

@@ -16,6 +16,7 @@ from app import (
)
from app.aws import s3
from app.celery import provider_tasks
from app.config import QueueNames
from app.dao.jobs_dao import (
dao_update_job,
dao_get_job_by_id,
@@ -80,7 +81,7 @@ def process_job(job_id):
process_row(row_number, recipient, personalisation, template, job, service)
if template.template_type == LETTER_TYPE:
build_dvla_file.apply_async([str(job.id)], queue='process-job')
build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
# temporary logging
current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id))
else:
@@ -112,12 +113,6 @@ def process_row(row_number, recipient, personalisation, template, job, service):
LETTER_TYPE: persist_letter
}
queues = {
SMS_TYPE: 'db-sms',
EMAIL_TYPE: 'db-email',
LETTER_TYPE: 'db-letter',
}
send_fn = send_fns[template_type]
send_fn.apply_async(
@@ -127,7 +122,7 @@ def process_row(row_number, recipient, personalisation, template, job, service):
encrypted,
datetime.utcnow().strftime(DATETIME_FORMAT)
),
queue=queues[template_type] if not service.research_mode else 'research-mode'
queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
)
@@ -181,7 +176,7 @@ def send_sms(self,
provider_tasks.deliver_sms.apply_async(
[str(saved_notification.id)],
queue='send-sms' if not service.research_mode else 'research-mode'
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info(
@@ -226,7 +221,7 @@ def send_email(self,
provider_tasks.deliver_email.apply_async(
[str(saved_notification.id)],
queue='send-email' if not service.research_mode else 'research-mode'
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at))
@@ -284,10 +279,9 @@ def build_dvla_file(self, job_id):
file_location="{}-dvla-job.text".format(job_id)
)
dao_update_job_status(job_id, JOB_STATUS_READY_TO_SEND)
notify_celery.send_task("aggregrate-dvla-files", ([str(job_id)], ), queue='aggregate-dvla-files')
else:
current_app.logger.info("All notifications for job {} are not persisted".format(job_id))
self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id))
self.retry(queue=QueueNames.RETRY, exc="All notifications for job {} are not persisted".format(job_id))
except Exception as e:
current_app.logger.exception("build_dvla_file threw exception")
raise e
@@ -341,7 +335,7 @@ def handle_exception(task, notification, notification_id, exc):
# send to the retry queue.
current_app.logger.exception('Retry' + retry_msg)
try:
task.retry(queue="retry", exc=exc)
task.retry(queue=QueueNames.RETRY, exc=exc)
except task.MaxRetriesExceededError:
current_app.logger.exception('Retry' + retry_msg)