darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-03 01:41:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into add-logging-to-sms-drs

Browse Source
This commit is contained in:
Rebecca Law
2017-07-19 14:30:51 +01:00
parent 49d1f52aef 92755c3417
commit 301339e689
26 changed files with 237 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/__init__.py
View File

@@ -49,7 +49,6 @@ def create_app(app_name=None):
from app.config import configs
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
application.config.from_object(configs[notify_environment])
if app_name:

27
app/celery/__init__.py
View File

@@ -0,0 +1,27 @@
class QueueNames(object):
PERIODIC = 'periodic-tasks'
PRIORITY = 'priority-tasks'
DATABASE = 'database-tasks'
SEND = 'send-tasks'
RESEARCH_MODE = 'research-mode-tasks'
STATISTICS = 'statistics-tasks'
JOBS = 'job-tasks'
RETRY = 'retry-tasks'
NOTIFY = 'notify-internal-tasks'
PROCESS_FTP = 'process-ftp-tasks'
@staticmethod
def all_queues():
return [
QueueNames.PRIORITY,
QueueNames.PERIODIC,
QueueNames.DATABASE,
QueueNames.SEND,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,
QueueNames.PROCESS_FTP
]

126
app/celery/celery.py
View File

@@ -1,11 +1,130 @@
from datetime import timedelta
from celery import Celery
from celery.schedules import crontab
from kombu import Queue, Exchange
from app.celery import QueueNames
class CeleryConfig:
def __init__(self, config):
self.broker_transport_options['queue_name_prefix'] = config['NOTIFICATION_QUEUE_PREFIX']
self.broker_url = config.get('BROKER_URL', 'sqs://')
broker_transport_options = {
'region': 'eu-west-1',
'polling_interval': 1, # 1 second
'visibility_timeout': 310,
'queue_name_prefix': None
}
enable_utc = True,
timezone = 'Europe/London'
accept_content = ['json']
task_serializer = 'json'
imports = ('app.celery.tasks', 'app.celery.scheduled_tasks')
beat_schedule = {
'run-scheduled-jobs': {
'task': 'run-scheduled-jobs',
'schedule': crontab(minute=1),
'options': {'queue': QueueNames.PERIODIC}
},
# 'send-scheduled-notifications': {
# 'task': 'send-scheduled-notifications',
# 'schedule': crontab(minute='*/15'),
# 'options': {'queue': 'periodic'}
# },
'delete-verify-codes': {
'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-invitations': {
'task': 'delete-invitations',
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(minute=20, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(minute=40, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(minute=0, hour=1),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(minute=0, hour=2),
'options': {'queue': QueueNames.PERIODIC}
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(minute=0, hour=3),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=0, hour=4),
'options': {'queue': QueueNames.PERIODIC},
# TODO: Avoid duplication of keywords - ideally by moving definitions out of models.py
'kwargs': {'job_types': ['email', 'sms']}
},
'remove_letter_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=20, hour=4),
'options': {'queue': QueueNames.PERIODIC},
# TODO: Avoid duplication of keywords - ideally by moving definitions out of models.py
'kwargs': {'job_types': ['letter']}
},
'remove_transformed_dvla_files': {
'task': 'remove_transformed_dvla_files',
'schedule': crontab(minute=40, hour=4),
'options': {'queue': QueueNames.PERIODIC}
},
'delete_dvla_response_files': {
'task': 'delete_dvla_response_files',
'schedule': crontab(minute=10, hour=5),
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
'schedule': crontab(minute=0, hour=5),
'options': {'queue': QueueNames.PERIODIC}
}
}
task_queues = []
class NotifyCelery(Celery):
def init_app(self, app):
super().__init__(app.import_name, broker=app.config['BROKER_URL'])
self.conf.update(app.config)
celery_config = CeleryConfig(app.config)
super().__init__(app.import_name, broker=celery_config.broker_url)
if app.config['INITIALISE_QUEUES']:
for queue in QueueNames.all_queues():
CeleryConfig.task_queues.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
self.config_from_object(celery_config)
TaskBase = self.Task
class ContextTask(TaskBase):
@@ -14,4 +133,5 @@ class NotifyCelery(Celery):
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
self.Task = ContextTask

2
app/celery/provider_tasks.py
View File

@@ -3,7 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
from sqlalchemy.orm.exc import NoResultFound
from app import notify_celery
from app.config import QueueNames
from app.celery import QueueNames
from app.dao import notifications_dao
from app.dao.notifications_dao import update_notification_status_by_id
from app.statsd_decorators import statsd

1
app/celery/research_mode_tasks.py
View File

@@ -1,7 +1,6 @@
import json
from flask import current_app
from app import notify_celery
from requests import request, RequestException, HTTPError
from app.models import SMS_TYPE

2
app/celery/scheduled_tasks.py
View File

@@ -28,7 +28,7 @@ from app.models import LETTER_TYPE
from app.notifications.process_notifications import send_notification_to_queue
from app.statsd_decorators import statsd
from app.celery.tasks import process_job
from app.config import QueueNames
from app.celery import QueueNames
@notify_celery.task(name="remove_csv_files")

2
app/celery/statistics_tasks.py
View File

@@ -10,7 +10,7 @@ from app.dao.statistics_dao import (
)
from app.dao.notifications_dao import get_notification_by_id
from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
from app.config import QueueNames
from app.celery import QueueNames
def create_initial_notification_statistic_tasks(notification):

2
app/celery/tasks.py
View File

@@ -19,8 +19,8 @@ from app import (
)
from app.aws import s3
from app.celery import provider_tasks
from app.config import QueueNames
from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
from app.celery import QueueNames
from app.dao.jobs_dao import (
dao_update_job,
dao_get_job_by_id,

144
app/config.py
View File

@@ -1,10 +1,6 @@
from datetime import timedelta
import os
import json
from celery.schedules import crontab
from kombu import Exchange, Queue
from app.models import (
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
@@ -18,34 +14,6 @@ if os.environ.get('VCAP_SERVICES'):
extract_cloudfoundry_config()
class QueueNames(object):
PERIODIC = 'periodic-tasks'
PRIORITY = 'priority-tasks'
DATABASE = 'database-tasks'
SEND = 'send-tasks'
RESEARCH_MODE = 'research-mode-tasks'
STATISTICS = 'statistics-tasks'
JOBS = 'job-tasks'
RETRY = 'retry-tasks'
NOTIFY = 'notify-internal-tasks'
PROCESS_FTP = 'process-ftp-tasks'
@staticmethod
def all_queues():
return [
QueueNames.PRIORITY,
QueueNames.PERIODIC,
QueueNames.DATABASE,
QueueNames.SEND,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,
QueueNames.PROCESS_FTP
]
class Config(object):
# URL of admin app
ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL']
@@ -126,104 +94,6 @@ class Config(object):
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
BROKER_URL = 'sqs://'
BROKER_TRANSPORT_OPTIONS = {
'region': AWS_REGION,
'polling_interval': 1, # 1 second
'visibility_timeout': 310,
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
}
CELERY_ENABLE_UTC = True,
CELERY_TIMEZONE = 'Europe/London'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks')
CELERYBEAT_SCHEDULE = {
'run-scheduled-jobs': {
'task': 'run-scheduled-jobs',
'schedule': crontab(minute=1),
'options': {'queue': QueueNames.PERIODIC}
},
# 'send-scheduled-notifications': {
# 'task': 'send-scheduled-notifications',
# 'schedule': crontab(minute='*/15'),
# 'options': {'queue': 'periodic'}
# },
'delete-verify-codes': {
'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-invitations': {
'task': 'delete-invitations',
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(minute=20, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(minute=40, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(minute=0, hour=1),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(minute=0, hour=2),
'options': {'queue': QueueNames.PERIODIC}
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(minute=0, hour=3),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=0, hour=4),
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]}
},
'remove_letter_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=20, hour=4),
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [LETTER_TYPE]}
},
'remove_transformed_dvla_files': {
'task': 'remove_transformed_dvla_files',
'schedule': crontab(minute=40, hour=4),
'options': {'queue': QueueNames.PERIODIC}
},
'delete_dvla_response_files': {
'task': 'delete_dvla_response_files',
'schedule': crontab(minute=10, hour=5),
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
'schedule': crontab(minute=0, hour=5),
'options': {'queue': QueueNames.PERIODIC}
}
}
CELERY_QUEUES = []
NOTIFICATIONS_ALERT = 5 # five mins
FROM_NUMBER = 'development'
@@ -262,6 +132,7 @@ class Config(object):
}
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
INITIALISE_QUEUES = False
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
@@ -271,24 +142,20 @@ class Config(object):
######################
class Development(Config):
INITIALISE_QUEUES = True
SQLALCHEMY_ECHO = False
NOTIFY_EMAIL_DOMAIN = 'notify.tools'
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
NOTIFY_ENVIRONMENT = 'development'
NOTIFICATION_QUEUE_PREFIX = 'development'
DEBUG = True
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_HOST_NAME = "http://localhost:6011"
API_RATE_LIMIT_ENABLED = True
class Test(Config):
INITIALISE_QUEUES = True
NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
FROM_NUMBER = 'testing'
NOTIFY_ENVIRONMENT = 'test'
@@ -302,11 +169,6 @@ class Test(Config):
BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011"

2
app/delivery/rest.py
View File

@@ -1,6 +1,6 @@
from flask import Blueprint, jsonify
from app.config import QueueNames
from app.celery import QueueNames
from app.delivery import send_to_providers
from app.models import EMAIL_TYPE
from app.celery import provider_tasks

2
app/invite/rest.py
View File

@@ -4,7 +4,7 @@ from flask import (
jsonify,
current_app)
from app.config import QueueNames
from app.celery import QueueNames
from app.dao.invited_user_dao import (
save_invited_user,
get_invited_user,

2
app/job/rest.py
View File

@@ -36,7 +36,7 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC
from app.utils import pagination_links
from app.config import QueueNames
from app.celery import QueueNames
job_blueprint = Blueprint('job', __name__, url_prefix='/service/<uuid:service_id>/job')

2
app/letters/send_letter_jobs.py
View File

@@ -2,7 +2,7 @@ from flask import Blueprint, jsonify
from flask import request
from app import notify_celery
from app.config import QueueNames
from app.celery import QueueNames
from app.dao.jobs_dao import dao_get_all_letter_jobs
from app.schemas import job_schema
from app.v2.errors import register_errors

2
app/notifications/notifications_letter_callback.py
View File

@@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses
from app.v2.errors import register_errors
from app.notifications.utils import autoconfirm_subscription
from app.schema_validation import validate
from app.config import QueueNames
from app.celery import QueueNames
letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
register_errors(letter_callback_blueprint)

2
app/notifications/process_notifications.py
View File

@@ -12,7 +12,7 @@ from app import redis_store
from app.celery import provider_tasks
from notifications_utils.clients import redis
from app.config import QueueNames
from app.celery import QueueNames
from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification
from app.dao.notifications_dao import (dao_create_notification,
dao_delete_notifications_and_history_by_id,

2
app/notifications/receive_notifications.py
View File

@@ -6,7 +6,7 @@ from notifications_utils.recipients import validate_and_format_phone_number
from app import statsd_client, firetext_client, mmg_client
from app.celery import tasks
from app.config import QueueNames
from app.celery import QueueNames
from app.dao.services_dao import dao_fetch_services_by_sms_sender
from app.dao.inbound_sms_dao import dao_create_inbound_sms
from app.models import InboundSms, INBOUND_SMS_TYPE, SMS_TYPE

2
app/notifications/rest.py
View File

@@ -6,7 +6,7 @@ from flask import (
)
from app import api_user, authenticated_service
from app.config import QueueNames
from app.celery import QueueNames
from app.dao import (
templates_dao,
notifications_dao

2
app/service/send_notification.py
View File

@@ -1,4 +1,4 @@
from app.config import QueueNames
from app.celery import QueueNames
from app.notifications.validators import (
check_service_over_daily_message_limit,
validate_and_format_recipient,

2
app/service/sender.py
View File

@@ -1,6 +1,6 @@
from flask import current_app
from app.config import QueueNames
from app.celery import QueueNames
from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service
from app.dao.templates_dao import dao_get_template_by_id
from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL

2
app/user/rest.py
View File

@@ -4,7 +4,7 @@ from datetime import datetime
from flask import (jsonify, request, Blueprint, current_app)
from app.config import QueueNames
from app.celery import QueueNames
from app.dao.users_dao import (
get_user_by_id,
save_model_user,

7
app/v2/notifications/post_notifications.py
View File

@@ -1,8 +1,8 @@
from flask import request, jsonify, current_app
from app import api_user, authenticated_service
from app.config import QueueNames
from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY, SCHEDULE_NOTIFICATIONS
from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
from app.celery import QueueNames
from app.notifications.process_notifications import (
persist_notification,
send_notification_to_queue,
@@ -11,20 +11,17 @@ from app.notifications.process_notifications import (
from app.notifications.validators import (
validate_and_format_recipient,
check_rate_limiting,
service_has_permission,
check_service_can_schedule_notification,
check_service_has_permission,
validate_template
)
from app.schema_validation import validate
from app.utils import get_public_notify_type_text
from app.v2.notifications import v2_notification_blueprint
from app.v2.notifications.notification_schemas import (
post_sms_request,
create_post_sms_response_from_notification,
post_email_request,
create_post_email_response_from_notification)
from app.v2.errors import BadRequestError
@v2_notification_blueprint.route('/<notification_type>', methods=['POST'])