darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-14 23:07:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

This massive set of changes uses the new queue names object throughout the app and tests.

Browse Source 
Lots of changes, all changing the line of code that puts things into queues, and the code that tests that.
This commit is contained in:
Martyn Inglis
2017-05-25 10:51:49 +01:00
parent 21586c917c
commit 2591d3a1df
28 changed files with 128 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/celery/provider_tasks.py
View File

@@ -3,6 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
from sqlalchemy.orm.exc import NoResultFound
from app import notify_celery
from app.config import QueueNames
from app.dao import notifications_dao
from app.dao.notifications_dao import update_notification_status_by_id
from app.statsd_decorators import statsd
@@ -46,7 +47,7 @@ def deliver_sms(self, notification_id):
current_app.logger.exception(
"SMS notification delivery for id: {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.exception(
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
@@ -70,7 +71,7 @@ def deliver_email(self, notification_id):
current_app.logger.exception(
"RETRY: Email notification {} failed".format(notification_id)
)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id)

3
app/celery/scheduled_tasks.py
View File

@@ -23,6 +23,7 @@ from app.dao.provider_details_dao import (
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
from app.statsd_decorators import statsd
from app.celery.tasks import process_job
from app.config import QueueNames
@notify_celery.task(name="remove_csv_files")
@@ -39,7 +40,7 @@ def remove_csv_files():
def run_scheduled_jobs():
try:
for job in dao_set_scheduled_jobs_to_pending():
process_job.apply_async([str(job.id)], queue="process-job")
process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
current_app.logger.info("Job ID {} added to process job queue".format(job.id))
except SQLAlchemyError as e:
current_app.logger.exception("Failed to run scheduled jobs")

12
app/celery/statistics_tasks.py
View File

@@ -3,7 +3,6 @@ from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery
from flask import current_app
from app.models import JobStatistics
from app.statsd_decorators import statsd
from app.dao.statistics_dao import (
create_or_update_job_sending_statistics,
@@ -11,16 +10,17 @@ from app.dao.statistics_dao import (
)
from app.dao.notifications_dao import get_notification_by_id
from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
from app.config import QueueNames
def create_initial_notification_statistic_tasks(notification):
if notification.job_id and notification.status:
record_initial_job_statistics.apply_async((str(notification.id),), queue="statistics")
record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
def create_outcome_notification_statistic_tasks(notification):
if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
record_outcome_job_statistics.apply_async((str(notification.id),), queue="statistics")
record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
@notify_celery.task(bind=True, name='record_initial_job_statistics', max_retries=20, default_retry_delay=10)
@@ -35,7 +35,7 @@ def record_initial_job_statistics(self, notification_id):
raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
except SQLAlchemyError as e:
current_app.logger.exception(e)
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task record_initial_job_statistics failed for notification {}".format(
@@ -53,12 +53,12 @@ def record_outcome_job_statistics(self, notification_id):
if notification:
updated_count = update_job_stats_outcome_count(notification)
if updated_count == 0:
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
else:
raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
except SQLAlchemyError as e:
current_app.logger.exception(e)
self.retry(queue="retry")
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task update_job_stats_outcome_count failed for notification {}".format(

20
app/celery/tasks.py
View File

@@ -16,6 +16,7 @@ from app import (
)
from app.aws import s3
from app.celery import provider_tasks
from app.config import QueueNames
from app.dao.jobs_dao import (
dao_update_job,
dao_get_job_by_id,
@@ -80,7 +81,7 @@ def process_job(job_id):
process_row(row_number, recipient, personalisation, template, job, service)
if template.template_type == LETTER_TYPE:
build_dvla_file.apply_async([str(job.id)], queue='process-job')
build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
# temporary logging
current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id))
else:
@@ -112,12 +113,6 @@ def process_row(row_number, recipient, personalisation, template, job, service):
LETTER_TYPE: persist_letter
}
queues = {
SMS_TYPE: 'db-sms',
EMAIL_TYPE: 'db-email',
LETTER_TYPE: 'db-letter',
}
send_fn = send_fns[template_type]
send_fn.apply_async(
@@ -127,7 +122,7 @@ def process_row(row_number, recipient, personalisation, template, job, service):
encrypted,
datetime.utcnow().strftime(DATETIME_FORMAT)
),
queue=queues[template_type] if not service.research_mode else 'research-mode'
queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
)
@@ -181,7 +176,7 @@ def send_sms(self,
provider_tasks.deliver_sms.apply_async(
[str(saved_notification.id)],
queue='send-sms' if not service.research_mode else 'research-mode'
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info(
@@ -226,7 +221,7 @@ def send_email(self,
provider_tasks.deliver_email.apply_async(
[str(saved_notification.id)],
queue='send-email' if not service.research_mode else 'research-mode'
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at))
@@ -284,10 +279,9 @@ def build_dvla_file(self, job_id):
file_location="{}-dvla-job.text".format(job_id)
)
dao_update_job_status(job_id, JOB_STATUS_READY_TO_SEND)
notify_celery.send_task("aggregrate-dvla-files", ([str(job_id)], ), queue='aggregate-dvla-files')
else:
current_app.logger.info("All notifications for job {} are not persisted".format(job_id))
self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id))
self.retry(queue=QueueNames.RETRY, exc="All notifications for job {} are not persisted".format(job_id))
except Exception as e:
current_app.logger.exception("build_dvla_file threw exception")
raise e
@@ -341,7 +335,7 @@ def handle_exception(task, notification, notification_id, exc):
# send to the retry queue.
current_app.logger.exception('Retry' + retry_msg)
try:
task.retry(queue="retry", exc=exc)
task.retry(queue=QueueNames.RETRY, exc=exc)
except task.MaxRetriesExceededError:
current_app.logger.exception('Retry' + retry_msg)

11
app/delivery/rest.py
View File

@@ -1,5 +1,6 @@
from flask import Blueprint, jsonify
from app.config import QueueNames
from app.delivery import send_to_providers
from app.models import EMAIL_TYPE
from app.celery import provider_tasks
@@ -23,18 +24,16 @@ def send_notification_to_provider(notification_id):
send_response(
send_to_providers.send_email_to_provider,
provider_tasks.deliver_email,
notification,
'send-email')
notification)
else:
send_response(
send_to_providers.send_sms_to_provider,
provider_tasks.deliver_sms,
notification,
'send-sms')
notification)
return jsonify({}), 204
def send_response(send_call, task_call, notification, queue):
def send_response(send_call, task_call, notification):
try:
send_call(notification)
except Exception as e:
@@ -43,4 +42,4 @@ def send_response(send_call, task_call, notification, queue):
notification.id,
notification.notification_type),
e)
task_call.apply_async((str(notification.id)), queue=queue)
task_call.apply_async((str(notification.id)), queue=QueueNames.SEND)

3
app/invite/rest.py
View File

@@ -4,6 +4,7 @@ from flask import (
jsonify,
current_app)
from app.config import QueueNames
from app.dao.invited_user_dao import (
save_invited_user,
get_invited_user,
@@ -44,7 +45,7 @@ def create_invited_user(service_id):
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(saved_notification, False, queue="notify")
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify(data=invited_user_schema.dump(invited_user).data), 201

4
app/job/rest.py
View File

@@ -34,6 +34,8 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC
from app.utils import pagination_links
from app.config import QueueNames
job_blueprint = Blueprint('job', __name__, url_prefix='/service/<uuid:service_id>/job')
from app.errors import (
@@ -143,7 +145,7 @@ def create_job(service_id):
dao_create_job(job)
if job.job_status == JOB_STATUS_PENDING:
process_job.apply_async([str(job.id)], queue="process-job")
process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
job_json = job_schema.dump(job).data
job_json['statistics'] = []

3
app/letters/send_letter_jobs.py
View File

@@ -2,6 +2,7 @@ from flask import Blueprint, jsonify
from flask import request
from app import notify_celery
from app.config import QueueNames
from app.dao.jobs_dao import dao_get_all_letter_jobs
from app.schemas import job_schema
from app.v2.errors import register_errors
@@ -15,7 +16,7 @@ register_errors(letter_job)
@letter_job.route('/send-letter-jobs', methods=['POST'])
def send_letter_jobs():
job_ids = validate(request.get_json(), letter_job_ids)
notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue="process-ftp")
notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP)
return jsonify(data={"response": "Task created to send files to DVLA"}), 201

4
app/notifications/notifications_letter_callback.py
View File

@@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses
from app.v2.errors import register_errors
from app.notifications.utils import autoconfirm_subscription
from app.schema_validation import validate
from app.config import QueueNames
letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
register_errors(letter_callback_blueprint)
@@ -54,7 +54,7 @@ def process_letter_response():
filename = message['Records'][0]['s3']['object']['key']
current_app.logger.info('Received file from DVLA: {}'.format(filename))
current_app.logger.info('DVLA callback: Calling task to update letter notifications')
update_letter_notifications_statuses.apply_async([filename], queue='notify')
update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY)
return jsonify(
result="success", message="DVLA callback succeeded"

9
app/notifications/process_notifications.py
View File

@@ -10,6 +10,8 @@ from notifications_utils.recipients import (
from app import redis_store
from app.celery import provider_tasks
from notifications_utils.clients import redis
from app.config import QueueNames
from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id
from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE
from app.v2.errors import BadRequestError, SendNotificationToQueueError
@@ -90,12 +92,9 @@ def persist_notification(
def send_notification_to_queue(notification, research_mode, queue=None):
if research_mode or notification.key_type == KEY_TYPE_TEST:
queue = 'research-mode'
queue = QueueNames.RESEARCH_MODE
elif not queue:
if notification.notification_type == SMS_TYPE:
queue = 'send-sms'
if notification.notification_type == EMAIL_TYPE:
queue = 'send-email'
queue = QueueNames.SEND
if notification.notification_type == SMS_TYPE:
deliver_task = provider_tasks.deliver_sms

3
app/notifications/rest.py
View File

@@ -6,6 +6,7 @@ from flask import (
)
from app import api_user, authenticated_service
from app.config import QueueNames
from app.dao import (
templates_dao,
notifications_dao
@@ -134,7 +135,7 @@ def send_notification(notification_type):
key_type=api_user.key_type,
simulated=simulated)
if not simulated:
queue_name = 'priority' if template.process_type == PRIORITY else None
queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
send_notification_to_queue(notification=notification_model,
research_mode=authenticated_service.research_mode,
queue=queue_name)

3
app/service/sender.py
View File

@@ -1,5 +1,6 @@
from flask import current_app
from app.config import QueueNames
from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service
from app.dao.templates_dao import dao_get_template_by_id
from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL
@@ -24,7 +25,7 @@ def send_notification_to_service_users(service_id, template_id, personalisation=
api_key_id=None,
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(notification, False, queue='notify')
send_notification_to_queue(notification, False, queue=QueueNames.NOTIFY)
def _add_user_fields(user, personalisation, fields):

11
app/user/rest.py
View File

@@ -4,6 +4,7 @@ from datetime import datetime
from flask import (jsonify, request, Blueprint, current_app)
from app.config import QueueNames
from app.dao.users_dao import (
get_user_by_id,
save_model_user,
@@ -182,7 +183,7 @@ def send_user_sms_code(user_id):
# Assume that we never want to observe the Notify service's research mode
# setting for this notification - we still need to be able to log into the
# admin even if we're doing user research using this service:
send_notification_to_queue(saved_notification, False, queue='notify')
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify({}), 204
@@ -212,7 +213,7 @@ def send_user_confirm_new_email(user_id):
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(saved_notification, False, queue='notify')
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify({}), 204
@@ -239,7 +240,7 @@ def send_user_email_verification(user_id):
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(saved_notification, False, queue="notify")
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify({}), 204
@@ -265,7 +266,7 @@ def send_already_registered_email(user_id):
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(saved_notification, False, queue="notify")
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify({}), 204
@@ -327,7 +328,7 @@ def send_user_reset_password():
key_type=KEY_TYPE_NORMAL
)
send_notification_to_queue(saved_notification, False, queue="notify")
send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
return jsonify({}), 204

3
app/v2/notifications/post_notifications.py
View File

@@ -2,6 +2,7 @@ from flask import request, jsonify, current_app
from sqlalchemy.orm.exc import NoResultFound
from app import api_user, authenticated_service
from app.config import QueueNames
from app.dao import services_dao, templates_dao
from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
from app.notifications.process_notifications import (
@@ -57,7 +58,7 @@ def post_notification(notification_type):
simulated=simulated)
if not simulated:
queue_name = 'priority' if template.process_type == PRIORITY else None
queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
send_notification_to_queue(
notification=notification,
research_mode=authenticated_service.research_mode,