darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-02 17:48:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

add devcontainer configs and docker network orchestration

Browse Source
This commit is contained in:
Jim Moffet
2022-06-13 13:16:32 -07:00
parent e604385e0c
commit 59b72f4853
18 changed files with 388 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -69,6 +69,8 @@ target/
*.DS_Store *.DS_Store
environment.sh environment.sh
.envrc .envrc
.env
.env*
celerybeat-schedule celerybeat-schedule

20
app/celery/research_mode_tasks.py
View File

@@ -164,7 +164,7 @@ def ses_notification_callback(reference):
'processingTimeMillis': 2003, 'processingTimeMillis': 2003,
'recipients': ['success@simulator.amazonses.com'], 'recipients': ['success@simulator.amazonses.com'],
'remoteMtaIp': '123.123.123.123', 'remoteMtaIp': '123.123.123.123',
'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com', 'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
'smtpResponse': '250 2.6.0 Message received', 'smtpResponse': '250 2.6.0 Message received',
'timestamp': '2017-11-17T12:14:03.646Z' 'timestamp': '2017-11-17T12:14:03.646Z'
}, },
@@ -201,7 +201,7 @@ def ses_notification_callback(reference):
'messageId': reference, 'messageId': reference,
'sendingAccountId': '12341234', 'sendingAccountId': '12341234',
'source': '"TEST" <TEST@notify.works>', 'source': '"TEST" <TEST@notify.works>',
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
'sourceIp': '0.0.0.1', 'sourceIp': '0.0.0.1',
'timestamp': '2017-11-17T12:14:01.643Z' 'timestamp': '2017-11-17T12:14:01.643Z'
}, },
@@ -211,14 +211,14 @@ def ses_notification_callback(reference):
return { return {
'Type': 'Notification', 'Type': 'Notification',
'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a', 'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
'Subject': None, 'Subject': None,
'Message': json.dumps(ses_message_body), 'Message': json.dumps(ses_message_body),
'Timestamp': '2017-11-17T12:14:03.710Z', 'Timestamp': '2017-11-17T12:14:03.710Z',
'SignatureVersion': '1', 'SignatureVersion': '1',
'Signature': '[REDACTED]', 'Signature': '[REDACTED]',
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem', 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]', 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
'MessageAttributes': {} 'MessageAttributes': {}
} }
@@ -244,7 +244,7 @@ def _ses_bounce_callback(reference, bounce_type):
}], }],
'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000', 'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
'remoteMtaIp': '123.123.123.123', 'remoteMtaIp': '123.123.123.123',
'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com', 'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
'timestamp': '2017-11-17T12:14:05.131Z' 'timestamp': '2017-11-17T12:14:05.131Z'
}, },
'mail': { 'mail': {
@@ -280,7 +280,7 @@ def _ses_bounce_callback(reference, bounce_type):
'messageId': reference, 'messageId': reference,
'sendingAccountId': '12341234', 'sendingAccountId': '12341234',
'source': '"TEST" <TEST@notify.works>', 'source': '"TEST" <TEST@notify.works>',
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
'sourceIp': '0.0.0.1', 'sourceIp': '0.0.0.1',
'timestamp': '2017-11-17T12:14:03.000Z' 'timestamp': '2017-11-17T12:14:03.000Z'
}, },
@@ -289,13 +289,13 @@ def _ses_bounce_callback(reference, bounce_type):
return { return {
'Type': 'Notification', 'Type': 'Notification',
'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7', 'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
'Subject': None, 'Subject': None,
'Message': json.dumps(ses_message_body), 'Message': json.dumps(ses_message_body),
'Timestamp': '2017-11-17T12:14:05.149Z', 'Timestamp': '2017-11-17T12:14:05.149Z',
'SignatureVersion': '1', 'SignatureVersion': '1',
'Signature': '[REDACTED]', # noqa 'Signature': '[REDACTED]', # noqa
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem', 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]', 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
'MessageAttributes': {} 'MessageAttributes': {}
} }

2
app/clients/cbc_proxy.py
View File

@@ -39,7 +39,7 @@ class CBCProxyClient:
if app.config.get('CBC_PROXY_ENABLED'): if app.config.get('CBC_PROXY_ENABLED'):
self._lambda_client = boto3.client( self._lambda_client = boto3.client(
'lambda', 'lambda',
region_name='eu-west-2', region_name='us-west-2',
aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'], aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'], aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
) )

163
app/config.py
View File

@@ -1,7 +1,6 @@
import json import json
import os import os
from datetime import timedelta from datetime import timedelta
from celery.schedules import crontab from celery.schedules import crontab
from kombu import Exchange, Queue from kombu import Exchange, Queue
@@ -100,14 +99,19 @@ class Config(object):
DANGEROUS_SALT = os.getenv('DANGEROUS_SALT') DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
# DB conection string # DB conection string
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
# MMG API Key # MMG API Key
MMG_API_KEY = os.getenv('MMG_API_KEY') MMG_API_KEY = os.getenv('MMG_API_KEY')
# Firetext API Key # Firetext API Key
FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") FIRETEXT_API_KEY = os.getenv('FIRETEXT_API_KEY')
FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") FIRETEXT_INTERNATIONAL_API_KEY = os.getenv(
'FIRETEXT_INTERNATIONAL_API_KEY', 'placeholder'
)
# Prefix to identify queues in SQS # Prefix to identify queues in SQS
NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
@@ -137,7 +141,7 @@ class Config(object):
########################### ###########################
NOTIFY_ENVIRONMENT = 'development' NOTIFY_ENVIRONMENT = 'development'
AWS_REGION = 'eu-west-1' AWS_REGION = 'us-west-2'
INVITATION_EXPIRATION_DAYS = 2 INVITATION_EXPIRATION_DAYS = 2
NOTIFY_APP_NAME = 'api' NOTIFY_APP_NAME = 'api'
SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_RECORD_QUERIES = False
@@ -160,10 +164,7 @@ class Config(object):
CHECK_PROXY_HEADER = False CHECK_PROXY_HEADER = False
# these should always add up to 100% # these should always add up to 100%
SMS_PROVIDER_RESTING_POINTS = { SMS_PROVIDER_RESTING_POINTS = {'mmg': 50, 'firetext': 50}
'mmg': 50,
'firetext': 50
}
NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
@@ -179,10 +180,16 @@ class Config(object):
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573' ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573'
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0' TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0'
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb' TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb'
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1' REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = (
'a42f1d17-9404-46d5-a647-d013bdfca3e1'
)
MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = (
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' 'c20206d5-bf03-4002-9a90-37d5032d9e84'
)
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = (
'522b6657-5ca5-4368-a294-6b527703bd0b'
)
NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
@@ -205,80 +212,85 @@ class Config(object):
], ],
# this is overriden by the -Q command, but locally, we should read from all queues # this is overriden by the -Q command, but locally, we should read from all queues
'task_queues': [ 'task_queues': [
Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues() Queue(queue, Exchange('default'), routing_key=queue)
for queue in QueueNames.all_queues()
], ],
'beat_schedule': { 'beat_schedule': {
# app/celery/scheduled_tasks.py # app/celery/scheduled_tasks.py
'run-scheduled-jobs': { 'run-scheduled-jobs': {
'task': 'run-scheduled-jobs', 'task': 'run-scheduled-jobs',
'schedule': crontab(minute='0,15,30,45'), 'schedule': crontab(minute='0,15,30,45'),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'delete-verify-codes': { 'delete-verify-codes': {
'task': 'delete-verify-codes', 'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63), 'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'delete-invitations': { 'delete-invitations': {
'task': 'delete-invitations', 'task': 'delete-invitations',
'schedule': timedelta(minutes=66), 'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'switch-current-sms-provider-on-slow-delivery': { 'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery', 'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute 'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'check-job-status': { 'check-job-status': {
'task': 'check-job-status', 'task': 'check-job-status',
'schedule': crontab(), 'schedule': crontab(),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'tend-providers-back-to-middle': { 'tend-providers-back-to-middle': {
'task': 'tend-providers-back-to-middle', 'task': 'tend-providers-back-to-middle',
'schedule': crontab(minute='*/5'), 'schedule': crontab(minute='*/5'),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'check-for-missing-rows-in-completed-jobs': { 'check-for-missing-rows-in-completed-jobs': {
'task': 'check-for-missing-rows-in-completed-jobs', 'task': 'check-for-missing-rows-in-completed-jobs',
'schedule': crontab(minute='*/10'), 'schedule': crontab(minute='*/10'),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'replay-created-notifications': { 'replay-created-notifications': {
'task': 'replay-created-notifications', 'task': 'replay-created-notifications',
'schedule': crontab(minute='0, 15, 30, 45'), 'schedule': crontab(minute='0, 15, 30, 45'),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
# app/celery/nightly_tasks.py # app/celery/nightly_tasks.py
'timeout-sending-notifications': { 'timeout-sending-notifications': {
'task': 'timeout-sending-notifications', 'task': 'timeout-sending-notifications',
'schedule': crontab(hour=0, minute=5), 'schedule': crontab(hour=0, minute=5),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'create-nightly-billing': { 'create-nightly-billing': {
'task': 'create-nightly-billing', 'task': 'create-nightly-billing',
'schedule': crontab(hour=0, minute=15), 'schedule': crontab(hour=0, minute=15),
'options': {'queue': QueueNames.REPORTING} 'options': {'queue': QueueNames.REPORTING},
}, },
'create-nightly-notification-status': { 'create-nightly-notification-status': {
'task': 'create-nightly-notification-status', 'task': 'create-nightly-notification-status',
'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications' 'schedule': crontab(
'options': {'queue': QueueNames.REPORTING} hour=0, minute=30
), # after 'timeout-sending-notifications'
'options': {'queue': QueueNames.REPORTING},
}, },
'delete-notifications-older-than-retention': { 'delete-notifications-older-than-retention': {
'task': 'delete-notifications-older-than-retention', 'task': 'delete-notifications-older-than-retention',
'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' 'schedule': crontab(
'options': {'queue': QueueNames.REPORTING} hour=3, minute=0
), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.REPORTING},
}, },
'delete-inbound-sms': { 'delete-inbound-sms': {
'task': 'delete-inbound-sms', 'task': 'delete-inbound-sms',
'schedule': crontab(hour=1, minute=40), 'schedule': crontab(hour=1, minute=40),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'save-daily-notification-processing-time': { 'save-daily-notification-processing-time': {
'task': 'save-daily-notification-processing-time', 'task': 'save-daily-notification-processing-time',
'schedule': crontab(hour=2, minute=0), 'schedule': crontab(hour=2, minute=0),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'remove_sms_email_jobs': { 'remove_sms_email_jobs': {
'task': 'remove_sms_email_jobs', 'task': 'remove_sms_email_jobs',
@@ -294,51 +306,51 @@ class Config(object):
'check-if-letters-still-in-created': { 'check-if-letters-still-in-created': {
'task': 'check-if-letters-still-in-created', 'task': 'check-if-letters-still-in-created',
'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'check-if-letters-still-pending-virus-check': { 'check-if-letters-still-pending-virus-check': {
'task': 'check-if-letters-still-pending-virus-check', 'task': 'check-if-letters-still-pending-virus-check',
'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'raise-alert-if-letter-notifications-still-sending': { 'raise-alert-if-letter-notifications-still-sending': {
'task': 'raise-alert-if-letter-notifications-still-sending', 'task': 'raise-alert-if-letter-notifications-still-sending',
'schedule': crontab(hour=17, minute=00), 'schedule': crontab(hour=17, minute=00),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
# The collate-letter-pdf does assume it is called in an hour that BST does not make a # The collate-letter-pdf does assume it is called in an hour that BST does not make a
# difference to the truncate date which translates to the filename to process # difference to the truncate date which translates to the filename to process
'collate-letter-pdfs-to-be-sent': { 'collate-letter-pdfs-to-be-sent': {
'task': 'collate-letter-pdfs-to-be-sent', 'task': 'collate-letter-pdfs-to-be-sent',
'schedule': crontab(hour=17, minute=50), 'schedule': crontab(hour=17, minute=50),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'raise-alert-if-no-letter-ack-file': { 'raise-alert-if-no-letter-ack-file': {
'task': 'raise-alert-if-no-letter-ack-file', 'task': 'raise-alert-if-no-letter-ack-file',
'schedule': crontab(hour=23, minute=00), 'schedule': crontab(hour=23, minute=00),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'trigger-link-tests': { 'trigger-link-tests': {
'task': 'trigger-link-tests', 'task': 'trigger-link-tests',
'schedule': timedelta(minutes=15), 'schedule': timedelta(minutes=15),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'auto-expire-broadcast-messages': { 'auto-expire-broadcast-messages': {
'task': 'auto-expire-broadcast-messages', 'task': 'auto-expire-broadcast-messages',
'schedule': timedelta(minutes=5), 'schedule': timedelta(minutes=5),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
'remove-yesterdays-planned-tests-on-govuk-alerts': { 'remove-yesterdays-planned-tests-on-govuk-alerts': {
'task': 'remove-yesterdays-planned-tests-on-govuk-alerts', 'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
'schedule': crontab(hour=00, minute=00), 'schedule': crontab(hour=00, minute=00),
'options': {'queue': QueueNames.PERIODIC} 'options': {'queue': QueueNames.PERIODIC},
}, },
} },
} }
# we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
@@ -364,32 +376,53 @@ class Config(object):
FREE_SMS_TIER_FRAGMENT_COUNT = 250000 FREE_SMS_TIER_FRAGMENT_COUNT = 250000
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) FIRETEXT_INBOUND_SMS_AUTH = json.loads(
os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')
)
MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) MMG_INBOUND_SMS_USERNAME = json.loads(
os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')
)
ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '') ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '') ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') TEMPLATE_PREVIEW_API_HOST = os.environ.get(
TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') 'TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013'
)
TEMPLATE_PREVIEW_API_KEY = os.environ.get(
'TEMPLATE_PREVIEW_API_KEY', 'my-secret-key'
)
DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') DOCUMENT_DOWNLOAD_API_HOST = os.environ.get(
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') 'DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000'
)
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get(
'DOCUMENT_DOWNLOAD_API_KEY', 'auth-token'
)
# these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") MMG_URL = os.environ.get('MMG_URL', 'https://api.mmg.co.uk/jsonv2a/api.php')
FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") FIRETEXT_URL = os.environ.get(
SES_STUB_URL = os.environ.get("SES_STUB_URL") 'FIRETEXT_URL', 'https://www.firetext.co.uk/api/sendsms/json'
)
SES_STUB_URL = os.environ.get('SES_STUB_URL')
AWS_REGION = 'eu-west-1' AWS_REGION = 'us-west-2'
CBC_PROXY_ENABLED = True CBC_PROXY_ENABLED = True
CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '') CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '') CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get(
'CBC_PROXY_AWS_SECRET_ACCESS_KEY', ''
)
ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE} ENABLED_CBCS = {
BroadcastProvider.EE,
BroadcastProvider.THREE,
BroadcastProvider.O2,
BroadcastProvider.VODAFONE,
}
# as defined in api db migration 0331_add_broadcast_org.py # as defined in api db migration 0331_add_broadcast_org.py
BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
@@ -399,6 +432,7 @@ class Config(object):
# Config overrides ### # Config overrides ###
###################### ######################
class Development(Config): class Development(Config):
DEBUG = True DEBUG = True
SQLALCHEMY_ECHO = False SQLALCHEMY_ECHO = False
@@ -417,7 +451,7 @@ class Development(Config):
INTERNAL_CLIENT_API_KEYS = { INTERNAL_CLIENT_API_KEYS = {
Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'],
} }
SECRET_KEY = 'dev-notify-secret-key' SECRET_KEY = 'dev-notify-secret-key'
@@ -428,10 +462,13 @@ class Development(Config):
NOTIFY_ENVIRONMENT = 'development' NOTIFY_ENVIRONMENT = 'development'
NOTIFY_LOG_PATH = 'application.log' NOTIFY_LOG_PATH = 'application.log'
NOTIFY_EMAIL_DOMAIN = "notify.tools" NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'dispostable.com')
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api') SQLALCHEMY_DATABASE_URI = os.getenv(
REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') 'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0')
ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
@@ -466,17 +503,20 @@ class Test(Development):
LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
# this is overriden in jenkins and on cloudfoundry # this is overriden in jenkins and on cloudfoundry
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api') SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
CELERY = { CELERY = {
**Config.CELERY, **Config.CELERY,
'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://',
} }
ANTIVIRUS_ENABLED = True ANTIVIRUS_ENABLED = True
API_RATE_LIMIT_ENABLED = True API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011" API_HOST_NAME = 'http://localhost:6011'
SMS_INBOUND_WHITELIST = ['203.0.113.195'] SMS_INBOUND_WHITELIST = ['203.0.113.195']
FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
@@ -486,7 +526,10 @@ class Test(Development):
FIRETEXT_URL = 'https://example.com/firetext' FIRETEXT_URL = 'https://example.com/firetext'
CBC_PROXY_ENABLED = True CBC_PROXY_ENABLED = True
DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] DVLA_EMAIL_ADDRESSES = [
'success@simulator.amazonses.com',
'success+2@simulator.amazonses.com',
]
class Preview(Config): class Preview(Config):
@@ -569,5 +612,5 @@ configs = {
'production': Live, 'production': Live,
'staging': Staging, 'staging': Staging,
'preview': Preview, 'preview': Preview,
'sandbox': Sandbox 'sandbox': Sandbox,
} }

42
devcontainer-api/.devcontainer.json Normal file
View File

@@ -0,0 +1,42 @@
{
"name": "notification-api",
"dockerComposeFile": "../docker-compose.devcontainer.yml",
"service": "dev",
"workspaceFolder": "/workspace",
"shutdownAction": "stopCompose",
"remoteEnv": {
"PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence
},
"settings": {
"[python]": {
"editor.formatOnSave": true
},
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.pythonPath": "/usr/local/bin/python"
},
"features": {
"docker-from-docker": {
"version": "latest",
"moby": true
}
},
"extensions": [
"ms-python.black-formatter",
"donjayamanne.python-extension-pack",
"ms-azuretools.vscode-docker",
"ms-python.vscode-pylance",
"eamodio.gitlens",
"wholroyd.jinja",
"pmbenjamin.vscode-snyk",
"visualstudioexptteam.vscodeintellicode",
"yzhang.markdown-all-in-one",
"ms-ossdata.vscode-postgresql",
"GitHub.copilot",
"ms-vsliveshare.vsliveshare",
"mtxr.sqltools",
"mtxr.sqltools-driver-pg",
],
"postCreateCommand": "notify-dev-entrypoint.sh",
"remoteUser": "vscode"
}

40
devcontainer-api/Dockerfile Normal file
View File

@@ -0,0 +1,40 @@
FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.9
RUN apt-get update \
&& apt-get -y install --no-install-recommends \
apt-utils \
postgresql-client \
2>&1 \
&& apt-get -y install \
curl \
emacs \
exa \
fd-find \
git \
iproute2 \
less \
libsodium-dev \
lsb-release \
man-db \
manpages \
net-tools \
openssh-client \
procps \
sudo \
tldr \
unzip \
vim \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/*
# Upgrade pip
RUN pip install --upgrade pip
COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/
COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/
ENV SHELL /bin/zsh
EXPOSE 8000
EXPOSE 6011

18
devcontainer-api/initdb/notify-db-entrypoint.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
#
# The PostgreSQL Docker image has an extension mechanism that does
# not necessitate to override the entrypoint or main command. One
# simply has to copy a shell script into the
# /docker-entrypoint-initdb.d/ initialization folder.
###################################################################
# Notify database setup.
createdb --user=postgres notification_api
wall "The db container entrypoint setup is complete!"

43
devcontainer-api/scripts/notify-dev-entrypoint.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
###################################################################
# Define aliases
echo -e "\n\n# User's Aliases" >> ~/.zshrc
echo -e "alias fd=fdfind" >> ~/.zshrc
echo -e "alias l='ls -al --color'" >> ~/.zshrc
echo -e "alias ls='exa'" >> ~/.zshrc
echo -e "alias l='exa -alh'" >> ~/.zshrc
echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
# # Kubectl aliases and command autocomplete
# echo -e "alias k='kubectl'" >> ~/.zshrc
# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
cd /workspace
# Warm up git index prior to display status in prompt else it will
# be quite slow on every invocation of starship.
git status
make generate-version-file
pip3 install -r requirements.txt
pip3 install -r requirements_for_test.txt
# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
pip3 install virtualenv
# Upgrade schema of the notification_api database
flask db upgrade
# Run flask server
# make run-flask

43
devcontainer-api/scripts/notify-worker-entrypoint.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
###################################################################
# Define aliases
echo -e "\n\n# User's Aliases" >> ~/.zshrc
echo -e "alias fd=fdfind" >> ~/.zshrc
echo -e "alias l='ls -al --color'" >> ~/.zshrc
echo -e "alias ls='exa'" >> ~/.zshrc
echo -e "alias l='exa -alh'" >> ~/.zshrc
echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
# # Kubectl aliases and command autocomplete
# echo -e "alias k='kubectl'" >> ~/.zshrc
# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
cd /workspace
# Warm up git index prior to display status in prompt else it will
# be quite slow on every invocation of starship.
git status
make generate-version-file
pip3 install -r requirements.txt
pip3 install -r requirements_for_test.txt
# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
pip3 install virtualenv
# Update commit hash etc...
make generate-version-file
# start listening to the queues
# make run-celery

72
docker-compose.devcontainer.yml Normal file
View File

@@ -0,0 +1,72 @@
version: '3'
services:
db:
container_name: db
image: postgres:13.4
volumes:
- ./devcontainer-api/initdb:/docker-entrypoint-initdb.d
environment:
PGGSSENCMODE: disable
POSTGRES_USER: postgres
POSTGRES_PASSWORD: chummy
POSTGRES_HOST_AUTH_METHOD: trust
ports:
- "5432:5432"
expose:
- "5432"
command:
- "postgres"
- "-c"
- "listen_addresses=*"
restart: always
redis:
container_name: redis
image: redis:6.2
restart: always
command: redis-server --port 6380
ports:
- "6380:6380"
expose:
- "6380"
dev:
container_name: dev
image: dev-notification-api
build:
context: .
dockerfile: devcontainer-api/Dockerfile
env_file: .env
volumes:
- .:/workspace:cached
command: sleep infinity
ports:
- 6011:6011
expose:
- "6011"
depends_on:
- db
- redis
links:
- db
restart: always
worker:
container_name: worker
image: dev-notification-api
build:
context: .
dockerfile: devcontainer-api/Dockerfile
env_file: .env
volumes:
- .:/workspace:cached
command: sleep infinity
depends_on:
- db
- redis
- dev
links:
- db
networks:
default:
name: notify-network
external: true

10
migrations/versions/0025_notify_service_data.py
View File

@@ -25,18 +25,18 @@ def upgrade():
password = hashpw(str(uuid.uuid4())) password = hashpw(str(uuid.uuid4()))
op.get_bind() op.get_bind()
user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin) user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin)
VALUES ('{}', 'Notify service user', 'notify-service-user@digital.cabinet-office', '{}', 0,'{}', '+441234123412', 'active', False) VALUES ('{}', 'Notify service user', 'testsender@dispostable.com', '{}', 0,'{}', '+441234123412', 'active', False)
""" """
op.execute(user_insert.format(user_id, datetime.utcnow(), password)) op.execute(user_insert.format(user_id, datetime.utcnow(), password))
service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
'{}', 'notify@digital.cabinet-office.gov.uk', 1) '{}', 'testsender@dispostable.com', 1)
""" """
op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id)) op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id))
service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
'{}', 'notify@digital.cabinet-office.gov.uk', 1) '{}', 'testsender@dispostable.com', 1)
""" """
op.execute(service_insert.format(service_id, datetime.utcnow(), user_id)) op.execute(service_insert.format(service_id, datetime.utcnow(), user_id))
user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')""" user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')"""

4
migrations/versions/0070_fix_notify_user_email.py
View File

@@ -17,7 +17,7 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
op.execute(""" op.execute("""
UPDATE users UPDATE users
SET email_address = 'notify-service-user@digital.cabinet-office.gov.uk' SET email_address = 'testsender@dispostable.com'
WHERE email_address = 'notify-service-user@digital.cabinet-office' WHERE email_address = 'notify-service-user@digital.cabinet-office'
""") """)
@@ -26,5 +26,5 @@ def downgrade():
op.execute(""" op.execute("""
UPDATE users UPDATE users
SET email_address = 'notify-service-user@digital.cabinet-office' SET email_address = 'notify-service-user@digital.cabinet-office'
WHERE email_address = 'notify-service-user@digital.cabinet-office.gov.uk' WHERE email_address = 'testsender@dispostable.com'
""") """)

2
migrations/versions/0082_add_golive_template.py
View File

@@ -64,7 +64,7 @@ It’s only an emergency if:
* a 500 response code appears when you try to send messages using the API * a 500 response code appears when you try to send messages using the API
If you have one of these emergencies, email details to: If you have one of these emergencies, email details to:
ooh-gov-uk-notify-support@digital.cabinet-office.gov.uk testsender@dispostable.com
^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team. ^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.

2
migrations/versions/0130_service_email_reply_to_row.py
View File

@@ -22,7 +22,7 @@ def upgrade():
INSERT INTO service_email_reply_to INSERT INTO service_email_reply_to
(id, service_id, email_address, is_default, created_at) (id, service_id, email_address, is_default, created_at)
VALUES VALUES
('{}','{}', 'notify+1@digital.cabinet-office.gov.uk', 'f', NOW()) ('{}','{}', 'testsender@dispostable.com', 'f', NOW())
""".format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID)) """.format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID))

1
requirements.in
View File

@@ -23,6 +23,7 @@ cachetools==5.1.0
beautifulsoup4==4.11.1 beautifulsoup4==4.11.1
lxml==4.8.0 lxml==4.8.0
Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned
python-dotenv==0.20.0
notifications-python-client==6.3.0 notifications-python-client==6.3.0

2
requirements.txt
View File

@@ -77,6 +77,8 @@ docopt==0.6.2
# via notifications-python-client # via notifications-python-client
docutils==0.16 docutils==0.16
# via awscli # via awscli
python-dotenv==0.20.0
# via -r requirements.in
eventlet==0.33.1 eventlet==0.33.1
# via gunicorn # via gunicorn
flask==2.1.2 flask==2.1.2

2
scripts/run_app_paas.sh
View File

@@ -65,7 +65,7 @@ function start_application {
} }
function start_aws_logs_agent { function start_aws_logs_agent {
exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
AWSLOGS_AGENT_PID=$! AWSLOGS_AGENT_PID=$!
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
} }

2
scripts/run_multi_worker_app_paas.sh
View File

@@ -102,7 +102,7 @@ function start_application {
function start_aws_logs_agent { function start_aws_logs_agent {
echo "Starting aws logs agent..." echo "Starting aws logs agent..."
exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
AWSLOGS_AGENT_PID=$! AWSLOGS_AGENT_PID=$!
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
} }