mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-09 06:32:11 -05:00
433 lines
16 KiB
Python
433 lines
16 KiB
Python
import json
|
|
from datetime import datetime, timedelta
|
|
from os import getenv, path
|
|
|
|
from boto3 import Session
|
|
from celery.schedules import crontab
|
|
from kombu import Exchange, Queue
|
|
|
|
import notifications_utils
|
|
from app.clients import AWS_CLIENT_CONFIG
|
|
from app.cloudfoundry_config import cloud_config
|
|
|
|
|
|
class QueueNames(object):
|
|
PERIODIC = "periodic-tasks"
|
|
DATABASE = "database-tasks"
|
|
SEND_SMS = "send-sms-tasks"
|
|
CHECK_SMS = "check-sms_tasks"
|
|
SEND_EMAIL = "send-email-tasks"
|
|
REPORTING = "reporting-tasks"
|
|
JOBS = "job-tasks"
|
|
RETRY = "retry-tasks"
|
|
NOTIFY = "notify-internal-tasks"
|
|
CALLBACKS = "service-callbacks"
|
|
CALLBACKS_RETRY = "service-callbacks-retry"
|
|
SMS_CALLBACKS = "sms-callbacks"
|
|
ANTIVIRUS = "antivirus-tasks"
|
|
SAVE_API_EMAIL = "save-api-email-tasks"
|
|
SAVE_API_SMS = "save-api-sms-tasks"
|
|
|
|
@staticmethod
|
|
def all_queues():
|
|
return [
|
|
QueueNames.PERIODIC,
|
|
QueueNames.DATABASE,
|
|
QueueNames.SEND_SMS,
|
|
QueueNames.CHECK_SMS,
|
|
QueueNames.SEND_EMAIL,
|
|
QueueNames.REPORTING,
|
|
QueueNames.JOBS,
|
|
QueueNames.RETRY,
|
|
QueueNames.NOTIFY,
|
|
QueueNames.CALLBACKS,
|
|
QueueNames.CALLBACKS_RETRY,
|
|
QueueNames.SMS_CALLBACKS,
|
|
QueueNames.SAVE_API_EMAIL,
|
|
QueueNames.SAVE_API_SMS,
|
|
]
|
|
|
|
|
|
class TaskNames(object):
|
|
PROCESS_INCOMPLETE_JOBS = "process-incomplete-jobs"
|
|
SCAN_FILE = "scan-file"
|
|
|
|
|
|
session = Session(
|
|
aws_access_key_id=getenv("CSV_AWS_ACCESS_KEY_ID"),
|
|
aws_secret_access_key=getenv("CSV_AWS_SECRET_ACCESS_KEY"),
|
|
region_name=getenv("CSV_AWS_REGION"),
|
|
)
|
|
|
|
|
|
class Config(object):
|
|
NOTIFY_APP_NAME = "api"
|
|
DEFAULT_REDIS_EXPIRE_TIME = 4 * 24 * 60 * 60
|
|
NOTIFY_ENVIRONMENT = getenv("NOTIFY_ENVIRONMENT", "development")
|
|
# URL of admin app
|
|
ADMIN_BASE_URL = getenv("ADMIN_BASE_URL", "http://localhost:6012")
|
|
# URL of api app (on AWS this is the internal api endpoint)
|
|
API_HOST_NAME = getenv("API_HOST_NAME", "http://localhost:6011")
|
|
|
|
# Credentials
|
|
# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
|
|
# ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
|
|
ADMIN_CLIENT_ID = getenv("ADMIN_CLIENT_ID", "notify-admin")
|
|
INTERNAL_CLIENT_API_KEYS = json.loads(
|
|
getenv(
|
|
"INTERNAL_CLIENT_API_KEYS",
|
|
('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv("ADMIN_CLIENT_SECRET"))),
|
|
)
|
|
)
|
|
ALLOW_EXPIRED_API_TOKEN = False
|
|
# encyption secret/salt
|
|
SECRET_KEY = getenv("SECRET_KEY")
|
|
DANGEROUS_SALT = getenv("DANGEROUS_SALT")
|
|
ROUTE_SECRET_KEY_1 = getenv("ROUTE_SECRET_KEY_1", "dev-route-secret-key-1")
|
|
ROUTE_SECRET_KEY_2 = getenv("ROUTE_SECRET_KEY_2", "dev-route-secret-key-2")
|
|
|
|
# DB settings
|
|
SQLALCHEMY_DATABASE_URI = cloud_config.database_url
|
|
SQLALCHEMY_RECORD_QUERIES = False
|
|
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
|
SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 40))
|
|
SQLALCHEMY_POOL_TIMEOUT = 30
|
|
SQLALCHEMY_POOL_RECYCLE = 300
|
|
SQLALCHEMY_STATEMENT_TIMEOUT = 1200
|
|
PAGE_SIZE = 20
|
|
API_PAGE_SIZE = 250
|
|
REDIS_URL = cloud_config.redis_url
|
|
REDIS_ENABLED = getenv("REDIS_ENABLED", "1") == "1"
|
|
EXPIRE_CACHE_TEN_MINUTES = 600
|
|
EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
|
|
|
|
# AWS Settings
|
|
AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER")
|
|
# Whether to ignore POSTs from SNS for replies to SMS we sent
|
|
RECEIVE_INBOUND_SMS = False
|
|
NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
|
|
SES_STUB_URL = (
|
|
None # TODO: set to a URL in env and remove this to use a stubbed SES service
|
|
)
|
|
# AWS SNS topics for delivery receipts
|
|
VALIDATE_SNS_TOPICS = True
|
|
VALID_SNS_TOPICS = cloud_config.sns_topic_arns
|
|
|
|
# these should always add up to 100%
|
|
SMS_PROVIDER_RESTING_POINTS = {
|
|
"sns": 100,
|
|
}
|
|
|
|
# Zendesk
|
|
ZENDESK_API_KEY = getenv("ZENDESK_API_KEY")
|
|
|
|
# Logging
|
|
DEBUG = False
|
|
|
|
# Antivirus
|
|
ANTIVIRUS_ENABLED = getenv("ANTIVIRUS_ENABLED", "1") == "1"
|
|
|
|
SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days
|
|
INVITATION_EXPIRATION_DAYS = 2
|
|
TEST_MESSAGE_FILENAME = "Test message"
|
|
ONE_OFF_MESSAGE_FILENAME = "Report"
|
|
MAX_VERIFY_CODE_COUNT = 5
|
|
MAX_FAILED_LOGIN_COUNT = 10
|
|
API_RATE_LIMIT_ENABLED = True
|
|
|
|
# Default data
|
|
CONFIG_FILES = path.dirname(__file__) + "/config_files/"
|
|
|
|
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
|
|
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
|
|
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
|
|
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
|
|
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
|
|
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
|
|
PASSWORD_RESET_TEMPLATE_ID = (
|
|
"474e9242-823b-4f99-813d-ed392e7f1201" # nosec B105 - this is not a password
|
|
)
|
|
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb"
|
|
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884"
|
|
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
|
|
ORGANIZATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
|
|
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
|
|
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"
|
|
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = (
|
|
"a42f1d17-9404-46d5-a647-d013bdfca3e1"
|
|
)
|
|
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
|
|
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = (
|
|
"c20206d5-bf03-4002-9a90-37d5032d9e84"
|
|
)
|
|
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = (
|
|
"522b6657-5ca5-4368-a294-6b527703bd0b"
|
|
)
|
|
NOTIFY_INTERNATIONAL_SMS_SENDER = getenv("AWS_US_TOLL_FREE_NUMBER")
|
|
LETTERS_VOLUME_EMAIL_TEMPLATE_ID = "11fad854-fd38-4a7c-bd17-805fb13dfc12"
|
|
NHS_EMAIL_BRANDING_ID = "a7dc4e56-660b-4db7-8cff-12c37b12b5ea"
|
|
# we only need real email in Live environment (production)
|
|
DVLA_EMAIL_ADDRESSES = json.loads(getenv("DVLA_EMAIL_ADDRESSES", "[]"))
|
|
|
|
current_minute = (datetime.now().minute + 1) % 60
|
|
|
|
S3_CLIENT = session.client("s3")
|
|
S3_RESOURCE = session.resource("s3", config=AWS_CLIENT_CONFIG)
|
|
|
|
CELERY = {
|
|
"broker_connection_retry_on_startup": True,
|
|
"worker_max_tasks_per_child": 500,
|
|
"task_ignore_result": True,
|
|
"result_persistent": False,
|
|
"broker_url": REDIS_URL,
|
|
"broker_transport_options": {
|
|
"visibility_timeout": 310,
|
|
},
|
|
"timezone": getenv("TIMEZONE", "UTC"),
|
|
"imports": [
|
|
"app.celery.tasks",
|
|
"app.celery.scheduled_tasks",
|
|
"app.celery.reporting_tasks",
|
|
"app.celery.nightly_tasks",
|
|
],
|
|
# this is overriden by the -Q command, but locally, we should read from all queues
|
|
"task_queues": [
|
|
Queue(queue, Exchange("default"), routing_key=queue)
|
|
for queue in QueueNames.all_queues()
|
|
],
|
|
"beat_schedule": {
|
|
# app/celery/scheduled_tasks.py
|
|
"run-scheduled-jobs": {
|
|
"task": "run-scheduled-jobs",
|
|
"schedule": crontab(minute="0,15,30,45"),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"delete-verify-codes": {
|
|
"task": "delete-verify-codes",
|
|
"schedule": timedelta(minutes=63),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"process-delivery-receipts": {
|
|
"task": "process-delivery-receipts",
|
|
"schedule": timedelta(minutes=2),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"cleanup-delivery-receipts": {
|
|
"task": "cleanup-delivery-receipts",
|
|
"schedule": timedelta(minutes=82),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"batch-insert-notifications": {
|
|
"task": "batch-insert-notifications",
|
|
"schedule": 10.0,
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"expire-or-delete-invitations": {
|
|
"task": "expire-or-delete-invitations",
|
|
"schedule": timedelta(minutes=66),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"check-job-status": {
|
|
"task": "check-job-status",
|
|
"schedule": crontab(),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"check-for-missing-rows-in-completed-jobs": {
|
|
"task": "check-for-missing-rows-in-completed-jobs",
|
|
"schedule": crontab(minute="*/10"),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"replay-created-notifications": {
|
|
"task": "replay-created-notifications",
|
|
"schedule": crontab(minute="0, 15, 30, 45"),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
# app/celery/nightly_tasks.py
|
|
"timeout-sending-notifications": {
|
|
"task": "timeout-sending-notifications",
|
|
"schedule": crontab(hour=4, minute=5),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"create-nightly-billing": {
|
|
"task": "create-nightly-billing",
|
|
"schedule": crontab(hour=4, minute=15),
|
|
"options": {"queue": QueueNames.REPORTING},
|
|
},
|
|
"create-nightly-notification-status": {
|
|
"task": "create-nightly-notification-status",
|
|
"schedule": crontab(
|
|
hour=4, minute=30
|
|
), # after 'timeout-sending-notifications'
|
|
"options": {"queue": QueueNames.REPORTING},
|
|
},
|
|
"delete-notifications-older-than-retention": {
|
|
"task": "delete-notifications-older-than-retention",
|
|
"schedule": crontab(
|
|
hour=7, minute=0
|
|
), # after 'create-nightly-notification-status'
|
|
"options": {"queue": QueueNames.REPORTING},
|
|
},
|
|
"delete-inbound-sms": {
|
|
"task": "delete-inbound-sms",
|
|
"schedule": crontab(hour=5, minute=40),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"save-daily-notification-processing-time": {
|
|
"task": "save-daily-notification-processing-time",
|
|
"schedule": crontab(hour=6, minute=0),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"delete_old_s3_objects": {
|
|
"task": "delete-old-s3-objects",
|
|
"schedule": crontab(hour=7, minute=10),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"regenerate-job-cache": {
|
|
"task": "regenerate-job-cache",
|
|
"schedule": crontab(minute="*/30"),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"generate-notifications-reports": {
|
|
"task": "generate-notifications-reports",
|
|
"schedule": crontab(hour=1, minute=0),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"regenerate-job-cache-on-startup": {
|
|
"task": "regenerate-job-cache",
|
|
"schedule": crontab(
|
|
minute=current_minute
|
|
), # Runs once at the next minute
|
|
"options": {
|
|
"queue": QueueNames.PERIODIC,
|
|
"expires": 60,
|
|
}, # Ensure it doesn't run if missed
|
|
},
|
|
"clean-job-cache": {
|
|
"task": "clean-job-cache",
|
|
"schedule": crontab(hour=2, minute=11),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"cleanup-unfinished-jobs": {
|
|
"task": "cleanup-unfinished-jobs",
|
|
"schedule": crontab(hour=4, minute=5),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"remove-sms-email-jobs": {
|
|
"task": "remove-sms-email-jobs",
|
|
"schedule": crontab(hour=8, minute=0),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
"check-for-services-with-high-failure-rates-or-sending-to-tv-numbers": {
|
|
"task": "check-for-services-with-high-failure-rates-or-sending-to-tv-numbers",
|
|
"schedule": crontab(day_of_week="mon-fri", hour=14, minute=30),
|
|
"options": {"queue": QueueNames.PERIODIC},
|
|
},
|
|
},
|
|
}
|
|
|
|
# we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
|
|
if getenv("CELERYD_PREFETCH_MULTIPLIER"):
|
|
CELERY["worker_prefetch_multiplier"] = getenv("CELERYD_PREFETCH_MULTIPLIER")
|
|
|
|
FROM_NUMBER = "development"
|
|
|
|
SIMULATED_EMAIL_ADDRESSES = (
|
|
"simulate-delivered@notifications.service.gov.uk",
|
|
"simulate-delivered-2@notifications.service.gov.uk",
|
|
"simulate-delivered-3@notifications.service.gov.uk",
|
|
)
|
|
# 7755 is success, 7167 is failure
|
|
SIMULATED_SMS_NUMBERS = ("+14254147755", "+14254147167")
|
|
|
|
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
|
|
|
|
TOTAL_MESSAGE_LIMIT = 5000000
|
|
|
|
DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT
|
|
|
|
HIGH_VOLUME_SERVICE = json.loads(getenv("HIGH_VOLUME_SERVICE", "[]"))
|
|
|
|
DOCUMENT_DOWNLOAD_API_HOST = getenv(
|
|
"DOCUMENT_DOWNLOAD_API_HOST", "http://localhost:7000"
|
|
)
|
|
DOCUMENT_DOWNLOAD_API_KEY = getenv("DOCUMENT_DOWNLOAD_API_KEY", "auth-token")
|
|
|
|
|
|
def _s3_credentials_from_env(bucket_prefix):
|
|
return {
|
|
"bucket": getenv(f"{bucket_prefix}_BUCKET_NAME"),
|
|
"access_key_id": getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"),
|
|
"secret_access_key": getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"),
|
|
"region": getenv(f"{bucket_prefix}_AWS_REGION"),
|
|
}
|
|
|
|
|
|
class Development(Config):
|
|
DEBUG = True
|
|
NOTIFY_LOG_LEVEL = "DEBUG"
|
|
SQLALCHEMY_ECHO = False
|
|
DVLA_EMAIL_ADDRESSES = ["success@simulator.amazonses.com"]
|
|
|
|
# Buckets
|
|
CSV_UPLOAD_BUCKET = _s3_credentials_from_env("CSV")
|
|
|
|
# credential overrides
|
|
DANGEROUS_SALT = "development-notify-salt"
|
|
SECRET_KEY = (
|
|
"dev-notify-secret-key" # nosec B105 - this is only used in development
|
|
)
|
|
INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ["dev-notify-secret-key"]}
|
|
ALLOW_EXPIRED_API_TOKEN = getenv("ALLOW_EXPIRED_API_TOKEN", "0") == "1"
|
|
|
|
|
|
class Test(Development):
|
|
FROM_NUMBER = "testing"
|
|
TESTING = True
|
|
ANTIVIRUS_ENABLED = True
|
|
DVLA_EMAIL_ADDRESSES = [
|
|
"success@simulator.amazonses.com",
|
|
"success+2@simulator.amazonses.com",
|
|
]
|
|
|
|
HIGH_VOLUME_SERVICE = [
|
|
"941b6f9a-50d7-4742-8d50-f365ca74bf27",
|
|
"63f95b86-2d19-4497-b8b2-ccf25457df4e",
|
|
"7e5950cb-9954-41f5-8376-962b8c8555cf",
|
|
"10d1b9c9-0072-4fa9-ae1c-595e333841da",
|
|
]
|
|
|
|
# this is overriden in CI
|
|
SQLALCHEMY_DATABASE_URI = getenv("SQLALCHEMY_DATABASE_TEST_URI")
|
|
|
|
CELERY = {
|
|
**Config.CELERY,
|
|
"broker_url": "you-forgot-to-mock-celery-in-your-tests://",
|
|
}
|
|
|
|
|
|
class Production(Config):
|
|
# buckets
|
|
CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
|
|
f"notify-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}"
|
|
)
|
|
|
|
FROM_NUMBER = "Notify.gov"
|
|
|
|
|
|
class Staging(Production):
|
|
pass
|
|
|
|
|
|
class Demo(Production):
|
|
pass
|
|
|
|
|
|
configs = {
|
|
"development": Development,
|
|
"test": Test,
|
|
"staging": Staging,
|
|
"demo": Demo,
|
|
"sandbox": Staging,
|
|
"production": Production,
|
|
}
|