darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-01-27 04:52:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

add devcontainer configs and docker network orchestration

Browse Source
This commit is contained in:
Jim Moffet
2022-06-13 13:16:32 -07:00
parent e604385e0c
commit 59b72f4853
18 changed files with 388 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -69,6 +69,8 @@ target/
*.DS_Store
environment.sh
.envrc
.env
.env*
celerybeat-schedule

20
app/celery/research_mode_tasks.py
View File

@@ -164,7 +164,7 @@ def ses_notification_callback(reference):
'processingTimeMillis': 2003,
'recipients': ['success@simulator.amazonses.com'],
'remoteMtaIp': '123.123.123.123',
'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
'smtpResponse': '250 2.6.0 Message received',
'timestamp': '2017-11-17T12:14:03.646Z'
},
@@ -201,7 +201,7 @@ def ses_notification_callback(reference):
'messageId': reference,
'sendingAccountId': '12341234',
'source': '"TEST" <TEST@notify.works>',
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
'sourceIp': '0.0.0.1',
'timestamp': '2017-11-17T12:14:01.643Z'
},
@@ -211,14 +211,14 @@ def ses_notification_callback(reference):
return {
'Type': 'Notification',
'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
'Subject': None,
'Message': json.dumps(ses_message_body),
'Timestamp': '2017-11-17T12:14:03.710Z',
'SignatureVersion': '1',
'Signature': '[REDACTED]',
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
'MessageAttributes': {}
}
@@ -244,7 +244,7 @@ def _ses_bounce_callback(reference, bounce_type):
}],
'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
'remoteMtaIp': '123.123.123.123',
'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
'timestamp': '2017-11-17T12:14:05.131Z'
},
'mail': {
@@ -280,7 +280,7 @@ def _ses_bounce_callback(reference, bounce_type):
'messageId': reference,
'sendingAccountId': '12341234',
'source': '"TEST" <TEST@notify.works>',
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
'sourceIp': '0.0.0.1',
'timestamp': '2017-11-17T12:14:03.000Z'
},
@@ -289,13 +289,13 @@ def _ses_bounce_callback(reference, bounce_type):
return {
'Type': 'Notification',
'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
'Subject': None,
'Message': json.dumps(ses_message_body),
'Timestamp': '2017-11-17T12:14:05.149Z',
'SignatureVersion': '1',
'Signature': '[REDACTED]', # noqa
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
'MessageAttributes': {}
}

2
app/clients/cbc_proxy.py
View File

@@ -39,7 +39,7 @@ class CBCProxyClient:
if app.config.get('CBC_PROXY_ENABLED'):
self._lambda_client = boto3.client(
'lambda',
region_name='eu-west-2',
region_name='us-west-2',
aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
)

163
app/config.py
View File

@@ -1,7 +1,6 @@
import json
import os
from datetime import timedelta
from celery.schedules import crontab
from kombu import Exchange, Queue
@@ -100,14 +99,19 @@ class Config(object):
DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
# DB conection string
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
# MMG API Key
MMG_API_KEY = os.getenv('MMG_API_KEY')
# Firetext API Key
FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
FIRETEXT_API_KEY = os.getenv('FIRETEXT_API_KEY')
FIRETEXT_INTERNATIONAL_API_KEY = os.getenv(
'FIRETEXT_INTERNATIONAL_API_KEY', 'placeholder'
)
# Prefix to identify queues in SQS
NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
@@ -137,7 +141,7 @@ class Config(object):
###########################
NOTIFY_ENVIRONMENT = 'development'
AWS_REGION = 'eu-west-1'
AWS_REGION = 'us-west-2'
INVITATION_EXPIRATION_DAYS = 2
NOTIFY_APP_NAME = 'api'
SQLALCHEMY_RECORD_QUERIES = False
@@ -160,10 +164,7 @@ class Config(object):
CHECK_PROXY_HEADER = False
# these should always add up to 100%
SMS_PROVIDER_RESTING_POINTS = {
'mmg': 50,
'firetext': 50
}
SMS_PROVIDER_RESTING_POINTS = {'mmg': 50, 'firetext': 50}
NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
@@ -179,10 +180,16 @@ class Config(object):
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573'
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0'
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb'
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1'
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = (
'a42f1d17-9404-46d5-a647-d013bdfca3e1'
)
MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = (
'c20206d5-bf03-4002-9a90-37d5032d9e84'
)
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = (
'522b6657-5ca5-4368-a294-6b527703bd0b'
)
NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
@@ -205,80 +212,85 @@ class Config(object):
],
# this is overriden by the -Q command, but locally, we should read from all queues
'task_queues': [
Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
Queue(queue, Exchange('default'), routing_key=queue)
for queue in QueueNames.all_queues()
],
'beat_schedule': {
# app/celery/scheduled_tasks.py
'run-scheduled-jobs': {
'task': 'run-scheduled-jobs',
'schedule': crontab(minute='0,15,30,45'),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'delete-verify-codes': {
'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'delete-invitations': {
'task': 'delete-invitations',
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'check-job-status': {
'task': 'check-job-status',
'schedule': crontab(),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'tend-providers-back-to-middle': {
'task': 'tend-providers-back-to-middle',
'schedule': crontab(minute='*/5'),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'check-for-missing-rows-in-completed-jobs': {
'task': 'check-for-missing-rows-in-completed-jobs',
'schedule': crontab(minute='*/10'),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'replay-created-notifications': {
'task': 'replay-created-notifications',
'schedule': crontab(minute='0, 15, 30, 45'),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
# app/celery/nightly_tasks.py
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(hour=0, minute=5),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'create-nightly-billing': {
'task': 'create-nightly-billing',
'schedule': crontab(hour=0, minute=15),
'options': {'queue': QueueNames.REPORTING}
'options': {'queue': QueueNames.REPORTING},
},
'create-nightly-notification-status': {
'task': 'create-nightly-notification-status',
'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications'
'options': {'queue': QueueNames.REPORTING}
'schedule': crontab(
hour=0, minute=30
), # after 'timeout-sending-notifications'
'options': {'queue': QueueNames.REPORTING},
},
'delete-notifications-older-than-retention': {
'task': 'delete-notifications-older-than-retention',
'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.REPORTING}
'schedule': crontab(
hour=3, minute=0
), # after 'create-nightly-notification-status'
'options': {'queue': QueueNames.REPORTING},
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(hour=1, minute=40),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'save-daily-notification-processing-time': {
'task': 'save-daily-notification-processing-time',
'schedule': crontab(hour=2, minute=0),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'remove_sms_email_jobs': {
'task': 'remove_sms_email_jobs',
@@ -294,51 +306,51 @@ class Config(object):
'check-if-letters-still-in-created': {
'task': 'check-if-letters-still-in-created',
'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'check-if-letters-still-pending-virus-check': {
'task': 'check-if-letters-still-pending-virus-check',
'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'raise-alert-if-letter-notifications-still-sending': {
'task': 'raise-alert-if-letter-notifications-still-sending',
'schedule': crontab(hour=17, minute=00),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
# The collate-letter-pdf does assume it is called in an hour that BST does not make a
# difference to the truncate date which translates to the filename to process
'collate-letter-pdfs-to-be-sent': {
'task': 'collate-letter-pdfs-to-be-sent',
'schedule': crontab(hour=17, minute=50),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'raise-alert-if-no-letter-ack-file': {
'task': 'raise-alert-if-no-letter-ack-file',
'schedule': crontab(hour=23, minute=00),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'trigger-link-tests': {
'task': 'trigger-link-tests',
'schedule': timedelta(minutes=15),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'auto-expire-broadcast-messages': {
'task': 'auto-expire-broadcast-messages',
'schedule': timedelta(minutes=5),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
'remove-yesterdays-planned-tests-on-govuk-alerts': {
'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
'schedule': crontab(hour=00, minute=00),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
},
}
},
}
# we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
@@ -364,32 +376,53 @@ class Config(object):
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
FIRETEXT_INBOUND_SMS_AUTH = json.loads(
os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')
)
MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
MMG_INBOUND_SMS_USERNAME = json.loads(
os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')
)
ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
TEMPLATE_PREVIEW_API_HOST = os.environ.get(
'TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013'
)
TEMPLATE_PREVIEW_API_KEY = os.environ.get(
'TEMPLATE_PREVIEW_API_KEY', 'my-secret-key'
)
DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000')
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
DOCUMENT_DOWNLOAD_API_HOST = os.environ.get(
'DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000'
)
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get(
'DOCUMENT_DOWNLOAD_API_KEY', 'auth-token'
)
# these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
SES_STUB_URL = os.environ.get("SES_STUB_URL")
MMG_URL = os.environ.get('MMG_URL', 'https://api.mmg.co.uk/jsonv2a/api.php')
FIRETEXT_URL = os.environ.get(
'FIRETEXT_URL', 'https://www.firetext.co.uk/api/sendsms/json'
)
SES_STUB_URL = os.environ.get('SES_STUB_URL')
AWS_REGION = 'eu-west-1'
AWS_REGION = 'us-west-2'
CBC_PROXY_ENABLED = True
CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '')
CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get(
'CBC_PROXY_AWS_SECRET_ACCESS_KEY', ''
)
ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE}
ENABLED_CBCS = {
BroadcastProvider.EE,
BroadcastProvider.THREE,
BroadcastProvider.O2,
BroadcastProvider.VODAFONE,
}
# as defined in api db migration 0331_add_broadcast_org.py
BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
@@ -399,6 +432,7 @@ class Config(object):
# Config overrides ###
######################
class Development(Config):
DEBUG = True
SQLALCHEMY_ECHO = False
@@ -417,7 +451,7 @@ class Development(Config):
INTERNAL_CLIENT_API_KEYS = {
Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'],
}
SECRET_KEY = 'dev-notify-secret-key'
@@ -428,10 +462,13 @@ class Development(Config):
NOTIFY_ENVIRONMENT = 'development'
NOTIFY_LOG_PATH = 'application.log'
NOTIFY_EMAIL_DOMAIN = "notify.tools"
NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'dispostable.com')
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api')
REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0')
ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
@@ -466,17 +503,20 @@ class Test(Development):
LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
# this is overriden in jenkins and on cloudfoundry
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
SQLALCHEMY_DATABASE_URI = os.getenv(
'SQLALCHEMY_DATABASE_URI',
'postgresql://postgres:chummy@db:5432/notification_api',
)
CELERY = {
**Config.CELERY,
'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
'broker_url': 'you-forgot-to-mock-celery-in-your-tests://',
}
ANTIVIRUS_ENABLED = True
API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011"
API_HOST_NAME = 'http://localhost:6011'
SMS_INBOUND_WHITELIST = ['203.0.113.195']
FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
@@ -486,7 +526,10 @@ class Test(Development):
FIRETEXT_URL = 'https://example.com/firetext'
CBC_PROXY_ENABLED = True
DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
DVLA_EMAIL_ADDRESSES = [
'success@simulator.amazonses.com',
'success+2@simulator.amazonses.com',
]
class Preview(Config):
@@ -569,5 +612,5 @@ configs = {
'production': Live,
'staging': Staging,
'preview': Preview,
'sandbox': Sandbox
'sandbox': Sandbox,
}

42
devcontainer-api/.devcontainer.json Normal file
View File

@@ -0,0 +1,42 @@
{
"name": "notification-api",
"dockerComposeFile": "../docker-compose.devcontainer.yml",
"service": "dev",
"workspaceFolder": "/workspace",
"shutdownAction": "stopCompose",
"remoteEnv": {
"PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence
},
"settings": {
"[python]": {
"editor.formatOnSave": true
},
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.pythonPath": "/usr/local/bin/python"
},
"features": {
"docker-from-docker": {
"version": "latest",
"moby": true
}
},
"extensions": [
"ms-python.black-formatter",
"donjayamanne.python-extension-pack",
"ms-azuretools.vscode-docker",
"ms-python.vscode-pylance",
"eamodio.gitlens",
"wholroyd.jinja",
"pmbenjamin.vscode-snyk",
"visualstudioexptteam.vscodeintellicode",
"yzhang.markdown-all-in-one",
"ms-ossdata.vscode-postgresql",
"GitHub.copilot",
"ms-vsliveshare.vsliveshare",
"mtxr.sqltools",
"mtxr.sqltools-driver-pg",
],
"postCreateCommand": "notify-dev-entrypoint.sh",
"remoteUser": "vscode"
}

40
devcontainer-api/Dockerfile Normal file
View File

@@ -0,0 +1,40 @@
FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.9
RUN apt-get update \
&& apt-get -y install --no-install-recommends \
apt-utils \
postgresql-client \
2>&1 \
&& apt-get -y install \
curl \
emacs \
exa \
fd-find \
git \
iproute2 \
less \
libsodium-dev \
lsb-release \
man-db \
manpages \
net-tools \
openssh-client \
procps \
sudo \
tldr \
unzip \
vim \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/*
# Upgrade pip
RUN pip install --upgrade pip
COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/
COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/
ENV SHELL /bin/zsh
EXPOSE 8000
EXPOSE 6011

18
devcontainer-api/initdb/notify-db-entrypoint.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
#
# The PostgreSQL Docker image has an extension mechanism that does
# not necessitate to override the entrypoint or main command. One
# simply has to copy a shell script into the
# /docker-entrypoint-initdb.d/ initialization folder.
###################################################################
# Notify database setup.
createdb --user=postgres notification_api
wall "The db container entrypoint setup is complete!"

43
devcontainer-api/scripts/notify-dev-entrypoint.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
###################################################################
# Define aliases
echo -e "\n\n# User's Aliases" >> ~/.zshrc
echo -e "alias fd=fdfind" >> ~/.zshrc
echo -e "alias l='ls -al --color'" >> ~/.zshrc
echo -e "alias ls='exa'" >> ~/.zshrc
echo -e "alias l='exa -alh'" >> ~/.zshrc
echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
# # Kubectl aliases and command autocomplete
# echo -e "alias k='kubectl'" >> ~/.zshrc
# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
cd /workspace
# Warm up git index prior to display status in prompt else it will
# be quite slow on every invocation of starship.
git status
make generate-version-file
pip3 install -r requirements.txt
pip3 install -r requirements_for_test.txt
# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
pip3 install virtualenv
# Upgrade schema of the notification_api database
flask db upgrade
# Run flask server
# make run-flask

43
devcontainer-api/scripts/notify-worker-entrypoint.sh Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
set -ex
###################################################################
# This script will get executed *once* the Docker container has
# been built. Commands that need to be executed with all available
# tools and the filesystem mount enabled should be located here.
###################################################################
# Define aliases
echo -e "\n\n# User's Aliases" >> ~/.zshrc
echo -e "alias fd=fdfind" >> ~/.zshrc
echo -e "alias l='ls -al --color'" >> ~/.zshrc
echo -e "alias ls='exa'" >> ~/.zshrc
echo -e "alias l='exa -alh'" >> ~/.zshrc
echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
# # Kubectl aliases and command autocomplete
# echo -e "alias k='kubectl'" >> ~/.zshrc
# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc
# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc
# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc
# echo -e "complete -F __start_kubectl k" >> ~/.zshrc
cd /workspace
# Warm up git index prior to display status in prompt else it will
# be quite slow on every invocation of starship.
git status
make generate-version-file
pip3 install -r requirements.txt
pip3 install -r requirements_for_test.txt
# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer
pip3 install virtualenv
# Update commit hash etc...
make generate-version-file
# start listening to the queues
# make run-celery

72
docker-compose.devcontainer.yml Normal file
View File

@@ -0,0 +1,72 @@
version: '3'
services:
db:
container_name: db
image: postgres:13.4
volumes:
- ./devcontainer-api/initdb:/docker-entrypoint-initdb.d
environment:
PGGSSENCMODE: disable
POSTGRES_USER: postgres
POSTGRES_PASSWORD: chummy
POSTGRES_HOST_AUTH_METHOD: trust
ports:
- "5432:5432"
expose:
- "5432"
command:
- "postgres"
- "-c"
- "listen_addresses=*"
restart: always
redis:
container_name: redis
image: redis:6.2
restart: always
command: redis-server --port 6380
ports:
- "6380:6380"
expose:
- "6380"
dev:
container_name: dev
image: dev-notification-api
build:
context: .
dockerfile: devcontainer-api/Dockerfile
env_file: .env
volumes:
- .:/workspace:cached
command: sleep infinity
ports:
- 6011:6011
expose:
- "6011"
depends_on:
- db
- redis
links:
- db
restart: always
worker:
container_name: worker
image: dev-notification-api
build:
context: .
dockerfile: devcontainer-api/Dockerfile
env_file: .env
volumes:
- .:/workspace:cached
command: sleep infinity
depends_on:
- db
- redis
- dev
links:
- db
networks:
default:
name: notify-network
external: true

10
migrations/versions/0025_notify_service_data.py
View File

@@ -25,18 +25,18 @@ def upgrade():
password = hashpw(str(uuid.uuid4()))
op.get_bind()
user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin)
VALUES ('{}', 'Notify service user', 'notify-service-user@digital.cabinet-office', '{}', 0,'{}', '+441234123412', 'active', False)
VALUES ('{}', 'Notify service user', 'testsender@dispostable.com', '{}', 0,'{}', '+441234123412', 'active', False)
"""
op.execute(user_insert.format(user_id, datetime.utcnow(), password))
service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk',
'{}', 'notify@digital.cabinet-office.gov.uk', 1)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
'{}', 'testsender@dispostable.com', 1)
"""
op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id))
service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk',
'{}', 'notify@digital.cabinet-office.gov.uk', 1)
VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com',
'{}', 'testsender@dispostable.com', 1)
"""
op.execute(service_insert.format(service_id, datetime.utcnow(), user_id))
user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')"""

4
migrations/versions/0070_fix_notify_user_email.py
View File

@@ -17,7 +17,7 @@ import sqlalchemy as sa
def upgrade():
op.execute("""
UPDATE users
SET email_address = 'notify-service-user@digital.cabinet-office.gov.uk'
SET email_address = 'testsender@dispostable.com'
WHERE email_address = 'notify-service-user@digital.cabinet-office'
""")
@@ -26,5 +26,5 @@ def downgrade():
op.execute("""
UPDATE users
SET email_address = 'notify-service-user@digital.cabinet-office'
WHERE email_address = 'notify-service-user@digital.cabinet-office.gov.uk'
WHERE email_address = 'testsender@dispostable.com'
""")

2
migrations/versions/0082_add_golive_template.py
View File

@@ -64,7 +64,7 @@ It’s only an emergency if:
* a 500 response code appears when you try to send messages using the API
If you have one of these emergencies, email details to:
ooh-gov-uk-notify-support@digital.cabinet-office.gov.uk
testsender@dispostable.com
^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.

2
migrations/versions/0130_service_email_reply_to_row.py
View File

@@ -22,7 +22,7 @@ def upgrade():
INSERT INTO service_email_reply_to
(id, service_id, email_address, is_default, created_at)
VALUES
('{}','{}', 'notify+1@digital.cabinet-office.gov.uk', 'f', NOW())
('{}','{}', 'testsender@dispostable.com', 'f', NOW())
""".format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID))

1
requirements.in
View File

@@ -23,6 +23,7 @@ cachetools==5.1.0
beautifulsoup4==4.11.1
lxml==4.8.0
Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned
python-dotenv==0.20.0
notifications-python-client==6.3.0

2
requirements.txt
View File

@@ -77,6 +77,8 @@ docopt==0.6.2
# via notifications-python-client
docutils==0.16
# via awscli
python-dotenv==0.20.0
# via -r requirements.in
eventlet==0.33.1
# via gunicorn
flask==2.1.2

2
scripts/run_app_paas.sh
View File

@@ -65,7 +65,7 @@ function start_application {
}
function start_aws_logs_agent {
exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
AWSLOGS_AGENT_PID=$!
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
}

2
scripts/run_multi_worker_app_paas.sh
View File

@@ -102,7 +102,7 @@ function start_application {
function start_aws_logs_agent {
echo "Starting aws logs agent..."
exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
AWSLOGS_AGENT_PID=$!
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
}