import json from datetime import timedelta from os import getenv from celery.schedules import crontab from kombu import Exchange, Queue from app.cloudfoundry_config import cloud_config class QueueNames(object): PERIODIC = 'periodic-tasks' PRIORITY = 'priority-tasks' DATABASE = 'database-tasks' SEND_SMS = 'send-sms-tasks' SEND_EMAIL = 'send-email-tasks' RESEARCH_MODE = 'research-mode-tasks' REPORTING = 'reporting-tasks' JOBS = 'job-tasks' RETRY = 'retry-tasks' NOTIFY = 'notify-internal-tasks' PROCESS_FTP = 'process-ftp-tasks' CREATE_LETTERS_PDF = 'create-letters-pdf-tasks' CALLBACKS = 'service-callbacks' CALLBACKS_RETRY = 'service-callbacks-retry' LETTERS = 'letter-tasks' SMS_CALLBACKS = 'sms-callbacks' ANTIVIRUS = 'antivirus-tasks' SANITISE_LETTERS = 'sanitise-letter-tasks' SAVE_API_EMAIL = 'save-api-email-tasks' SAVE_API_SMS = 'save-api-sms-tasks' @staticmethod def all_queues(): return [ QueueNames.PRIORITY, QueueNames.PERIODIC, QueueNames.DATABASE, QueueNames.SEND_SMS, QueueNames.SEND_EMAIL, QueueNames.RESEARCH_MODE, QueueNames.REPORTING, QueueNames.JOBS, QueueNames.RETRY, QueueNames.NOTIFY, QueueNames.CREATE_LETTERS_PDF, QueueNames.CALLBACKS, QueueNames.CALLBACKS_RETRY, QueueNames.LETTERS, QueueNames.SMS_CALLBACKS, QueueNames.SAVE_API_EMAIL, QueueNames.SAVE_API_SMS, ] class TaskNames(object): PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs' ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs' SCAN_FILE = 'scan-file' SANITISE_LETTER = 'sanitise-and-upload-letter' CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter' RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter' class Config(object): NOTIFY_APP_NAME = 'api' NOTIFY_ENVIRONMENT = getenv('NOTIFY_ENVIRONMENT', 'development') # URL of admin app ADMIN_BASE_URL = getenv('ADMIN_BASE_URL', 'http://localhost:6012') # URL of api app (on AWS this is the internal api endpoint) API_HOST_NAME = getenv('API_HOST_NAME', 'http://localhost:6011') # Credentials # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match ADMIN_CLIENT_ID = getenv('ADMIN_CLIENT_ID', 'notify-admin') INTERNAL_CLIENT_API_KEYS = json.loads( getenv( 'INTERNAL_CLIENT_API_KEYS', ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv('ADMIN_CLIENT_SECRET'))) ) ) # encyption secret/salt SECRET_KEY = getenv('SECRET_KEY') DANGEROUS_SALT = getenv('DANGEROUS_SALT') ROUTE_SECRET_KEY_1 = getenv('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') ROUTE_SECRET_KEY_2 = getenv('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') # DB settings SQLALCHEMY_DATABASE_URI = cloud_config.database_url SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_POOL_SIZE = int(getenv('SQLALCHEMY_POOL_SIZE', 5)) SQLALCHEMY_POOL_TIMEOUT = 30 SQLALCHEMY_POOL_RECYCLE = 300 SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 REDIS_URL = cloud_config.redis_url REDIS_ENABLED = getenv('REDIS_ENABLED', '0') == '1' EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 # AWS Settings AWS_REGION = getenv('AWS_REGION') AWS_PINPOINT_REGION = getenv("AWS_PINPOINT_REGION") AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER") # Whether to ignore POSTs from SNS for replies to SMS we sent RECEIVE_INBOUND_SMS = False NOTIFY_EMAIL_DOMAIN = getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # AWS SNS topics for delivery receipts VALIDATE_SNS_TOPICS = True VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] # SMS config to be cleaned up during https://github.com/GSA/notifications-api/issues/7 # MMG API Key MMG_API_KEY = getenv('MMG_API_KEY', 'placeholder') # Firetext API Key FIRETEXT_API_KEY = getenv("FIRETEXT_API_KEY", "placeholder") FIRETEXT_INTERNATIONAL_API_KEY = getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # these should always add up to 100% SMS_PROVIDER_RESTING_POINTS = { 'mmg': 50, 'firetext': 50 } FIRETEXT_INBOUND_SMS_AUTH = json.loads(getenv('FIRETEXT_INBOUND_SMS_AUTH', '[]')) MMG_INBOUND_SMS_AUTH = json.loads(getenv('MMG_INBOUND_SMS_AUTH', '[]')) MMG_INBOUND_SMS_USERNAME = json.loads(getenv('MMG_INBOUND_SMS_USERNAME', '[]')) MMG_URL = getenv("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") FIRETEXT_URL = getenv("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") # Zendesk ZENDESK_API_KEY = getenv('ZENDESK_API_KEY') # Logging DEBUG = False NOTIFY_LOG_PATH = getenv('NOTIFY_LOG_PATH', 'logs/application.log') # Monitoring CRONITOR_ENABLED = False CRONITOR_KEYS = json.loads(getenv('CRONITOR_KEYS', '{}')) STATSD_HOST = getenv('STATSD_HOST') STATSD_PORT = 8125 STATSD_ENABLED = bool(STATSD_HOST) # Antivirus ANTIVIRUS_ENABLED = getenv('ANTIVIRUS_ENABLED', '1') == '1' SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days INVITATION_EXPIRATION_DAYS = 2 TEST_MESSAGE_FILENAME = 'Test message' ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 5 MAX_FAILED_LOGIN_COUNT = 10 API_RATE_LIMIT_ENABLED = True # be careful increasing this size without being sure that we won't see slowness in pysftp MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb MAX_LETTER_PDF_COUNT_PER_ZIP = 500 NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc' SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246' EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164' NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27' PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb' CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884' SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde' ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573' TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0' TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb' REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1' MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' NOTIFY_INTERNATIONAL_SMS_SENDER = '18446120782' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' # we only need real email in Live environment (production) DVLA_EMAIL_ADDRESSES = json.loads(getenv('DVLA_EMAIL_ADDRESSES', '[]')) CELERY = { 'broker_url': REDIS_URL, 'broker_transport_options': { 'visibility_timeout': 310, }, 'timezone': 'Europe/London', 'imports': [ 'app.celery.tasks', 'app.celery.scheduled_tasks', 'app.celery.reporting_tasks', 'app.celery.nightly_tasks', ], # this is overriden by the -Q command, but locally, we should read from all queues 'task_queues': [ Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues() ], 'beat_schedule': { # app/celery/scheduled_tasks.py 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute='0,15,30,45'), 'options': {'queue': QueueNames.PERIODIC} }, 'delete-verify-codes': { 'task': 'delete-verify-codes', 'schedule': timedelta(minutes=63), 'options': {'queue': QueueNames.PERIODIC} }, 'delete-invitations': { 'task': 'delete-invitations', 'schedule': timedelta(minutes=66), 'options': {'queue': QueueNames.PERIODIC} }, 'switch-current-sms-provider-on-slow-delivery': { 'task': 'switch-current-sms-provider-on-slow-delivery', 'schedule': crontab(), # Every minute 'options': {'queue': QueueNames.PERIODIC} }, 'check-job-status': { 'task': 'check-job-status', 'schedule': crontab(), 'options': {'queue': QueueNames.PERIODIC} }, 'tend-providers-back-to-middle': { 'task': 'tend-providers-back-to-middle', 'schedule': crontab(minute='*/5'), 'options': {'queue': QueueNames.PERIODIC} }, 'check-for-missing-rows-in-completed-jobs': { 'task': 'check-for-missing-rows-in-completed-jobs', 'schedule': crontab(minute='*/10'), 'options': {'queue': QueueNames.PERIODIC} }, 'replay-created-notifications': { 'task': 'replay-created-notifications', 'schedule': crontab(minute='0, 15, 30, 45'), 'options': {'queue': QueueNames.PERIODIC} }, # app/celery/nightly_tasks.py 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(hour=0, minute=5), 'options': {'queue': QueueNames.PERIODIC} }, 'create-nightly-billing': { 'task': 'create-nightly-billing', 'schedule': crontab(hour=0, minute=15), 'options': {'queue': QueueNames.REPORTING} }, 'create-nightly-notification-status': { 'task': 'create-nightly-notification-status', 'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications' 'options': {'queue': QueueNames.REPORTING} }, 'delete-notifications-older-than-retention': { 'task': 'delete-notifications-older-than-retention', 'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' 'options': {'queue': QueueNames.REPORTING} }, 'delete-inbound-sms': { 'task': 'delete-inbound-sms', 'schedule': crontab(hour=1, minute=40), 'options': {'queue': QueueNames.PERIODIC} }, 'save-daily-notification-processing-time': { 'task': 'save-daily-notification-processing-time', 'schedule': crontab(hour=2, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, 'remove_sms_email_jobs': { 'task': 'remove_sms_email_jobs', 'schedule': crontab(hour=4, minute=0), 'options': {'queue': QueueNames.PERIODIC}, }, 'remove_letter_jobs': { 'task': 'remove_letter_jobs', 'schedule': crontab(hour=4, minute=20), # since we mark jobs as archived 'options': {'queue': QueueNames.PERIODIC}, }, 'check-if-letters-still-in-created': { 'task': 'check-if-letters-still-in-created', 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, 'check-if-letters-still-pending-virus-check': { 'task': 'check-if-letters-still-pending-virus-check', 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), 'options': {'queue': QueueNames.PERIODIC} }, 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), 'options': {'queue': QueueNames.PERIODIC} }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=17, minute=00), 'options': {'queue': QueueNames.PERIODIC} }, # The collate-letter-pdf does assume it is called in an hour that BST does not make a # difference to the truncate date which translates to the filename to process 'collate-letter-pdfs-to-be-sent': { 'task': 'collate-letter-pdfs-to-be-sent', 'schedule': crontab(hour=17, minute=50), 'options': {'queue': QueueNames.PERIODIC} }, 'raise-alert-if-no-letter-ack-file': { 'task': 'raise-alert-if-no-letter-ack-file', 'schedule': crontab(hour=23, minute=00), 'options': {'queue': QueueNames.PERIODIC} }, 'trigger-link-tests': { 'task': 'trigger-link-tests', 'schedule': timedelta(minutes=15), 'options': {'queue': QueueNames.PERIODIC} }, } } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks if getenv('CELERYD_PREFETCH_MULTIPLIER'): CELERY['worker_prefetch_multiplier'] = getenv('CELERYD_PREFETCH_MULTIPLIER') FROM_NUMBER = 'development' SIMULATED_EMAIL_ADDRESSES = ( 'simulate-delivered@notifications.service.gov.uk', 'simulate-delivered-2@notifications.service.gov.uk', 'simulate-delivered-3@notifications.service.gov.uk', ) SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') FREE_SMS_TIER_FRAGMENT_COUNT = 250000 HIGH_VOLUME_SERVICE = json.loads(getenv('HIGH_VOLUME_SERVICE', '[]')) TEMPLATE_PREVIEW_API_HOST = getenv('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') TEMPLATE_PREVIEW_API_KEY = getenv('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') DOCUMENT_DOWNLOAD_API_HOST = getenv('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') DOCUMENT_DOWNLOAD_API_KEY = getenv('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') def _default_s3_credentials(bucket_name): return { 'bucket': bucket_name, 'access_key_id': getenv('AWS_ACCESS_KEY_ID'), 'secret_access_key': getenv('AWS_SECRET_ACCESS_KEY'), 'region': getenv('AWS_REGION') } class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] # Buckets CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') # credential overrides DANGEROUS_SALT = 'dev-notify-salt' SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']} class Test(Development): FROM_NUMBER = 'testing' TESTING = True ANTIVIRUS_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] MMG_INBOUND_SMS_AUTH = ['testkey'] MMG_INBOUND_SMS_USERNAME = ['username'] HIGH_VOLUME_SERVICE = [ '941b6f9a-50d7-4742-8d50-f365ca74bf27', '63f95b86-2d19-4497-b8b2-ccf25457df4e', '7e5950cb-9954-41f5-8376-962b8c8555cf', '10d1b9c9-0072-4fa9-ae1c-595e333841da', ] CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload') CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') # this is overriden in CI SQLALCHEMY_DATABASE_URI = getenv('SQLALCHEMY_DATABASE_TEST_URI') CELERY = { **Config.CELERY, 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' } TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999' class Production(Config): # buckets CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}") CONTACT_LIST_BUCKET = cloud_config.s3_credentials( f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}") FROM_NUMBER = 'US Notify' CRONITOR_ENABLED = True class Staging(Production): pass class Demo(Production): pass configs = { 'development': Development, 'test': Test, 'staging': Staging, 'demo': Demo, 'production': Production }