From 59b72f4853c3e9099c12c8289456bd65bf1d4628 Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 13:16:32 -0700 Subject: [PATCH] add devcontainer configs and docker network orchestration --- .gitignore | 2 + app/celery/research_mode_tasks.py | 20 +-- app/clients/cbc_proxy.py | 2 +- app/config.py | 163 +++++++++++------- devcontainer-api/.devcontainer.json | 42 +++++ devcontainer-api/Dockerfile | 40 +++++ .../initdb/notify-db-entrypoint.sh | 18 ++ .../scripts/notify-dev-entrypoint.sh | 43 +++++ .../scripts/notify-worker-entrypoint.sh | 43 +++++ docker-compose.devcontainer.yml | 72 ++++++++ .../versions/0025_notify_service_data.py | 10 +- .../versions/0070_fix_notify_user_email.py | 4 +- .../versions/0082_add_golive_template.py | 2 +- .../0130_service_email_reply_to_row.py | 2 +- requirements.in | 1 + requirements.txt | 2 + scripts/run_app_paas.sh | 2 +- scripts/run_multi_worker_app_paas.sh | 2 +- 18 files changed, 388 insertions(+), 82 deletions(-) create mode 100644 devcontainer-api/.devcontainer.json create mode 100644 devcontainer-api/Dockerfile create mode 100755 devcontainer-api/initdb/notify-db-entrypoint.sh create mode 100755 devcontainer-api/scripts/notify-dev-entrypoint.sh create mode 100755 devcontainer-api/scripts/notify-worker-entrypoint.sh create mode 100644 docker-compose.devcontainer.yml diff --git a/.gitignore b/.gitignore index 4ae3c89a8..b5c8d94af 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ target/ *.DS_Store environment.sh .envrc +.env +.env* celerybeat-schedule diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index db68a2fe4..e937739f5 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -164,7 +164,7 @@ def ses_notification_callback(reference): 'processingTimeMillis': 2003, 'recipients': ['success@simulator.amazonses.com'], 'remoteMtaIp': '123.123.123.123', - 'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com', + 'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com', 'smtpResponse': '250 2.6.0 Message received', 'timestamp': '2017-11-17T12:14:03.646Z' }, @@ -201,7 +201,7 @@ def ses_notification_callback(reference): 'messageId': reference, 'sendingAccountId': '12341234', 'source': '"TEST" ', - 'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', + 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works', 'sourceIp': '0.0.0.1', 'timestamp': '2017-11-17T12:14:01.643Z' }, @@ -211,14 +211,14 @@ def ses_notification_callback(reference): return { 'Type': 'Notification', 'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a', - 'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', + 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications', 'Subject': None, 'Message': json.dumps(ses_message_body), 'Timestamp': '2017-11-17T12:14:03.710Z', 'SignatureVersion': '1', 'Signature': '[REDACTED]', - 'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem', - 'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]', + 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem', + 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]', 'MessageAttributes': {} } @@ -244,7 +244,7 @@ def _ses_bounce_callback(reference, bounce_type): }], 'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000', 'remoteMtaIp': '123.123.123.123', - 'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com', + 'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com', 'timestamp': '2017-11-17T12:14:05.131Z' }, 'mail': { @@ -280,7 +280,7 @@ def _ses_bounce_callback(reference, bounce_type): 'messageId': reference, 'sendingAccountId': '12341234', 'source': '"TEST" ', - 'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', + 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works', 'sourceIp': '0.0.0.1', 'timestamp': '2017-11-17T12:14:03.000Z' }, @@ -289,13 +289,13 @@ def _ses_bounce_callback(reference, bounce_type): return { 'Type': 'Notification', 'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7', - 'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', + 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications', 'Subject': None, 'Message': json.dumps(ses_message_body), 'Timestamp': '2017-11-17T12:14:05.149Z', 'SignatureVersion': '1', 'Signature': '[REDACTED]', # noqa - 'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem', - 'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]', + 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem', + 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]', 'MessageAttributes': {} } diff --git a/app/clients/cbc_proxy.py b/app/clients/cbc_proxy.py index 89ec65db3..a33b0b284 100644 --- a/app/clients/cbc_proxy.py +++ b/app/clients/cbc_proxy.py @@ -39,7 +39,7 @@ class CBCProxyClient: if app.config.get('CBC_PROXY_ENABLED'): self._lambda_client = boto3.client( 'lambda', - region_name='eu-west-2', + region_name='us-west-2', aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'], aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'], ) diff --git a/app/config.py b/app/config.py index 35bc7a095..fd90ed67e 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,6 @@ import json import os from datetime import timedelta - from celery.schedules import crontab from kombu import Exchange, Queue @@ -100,14 +99,19 @@ class Config(object): DANGEROUS_SALT = os.getenv('DANGEROUS_SALT') # DB conection string - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) # MMG API Key MMG_API_KEY = os.getenv('MMG_API_KEY') # Firetext API Key - FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") - FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") + FIRETEXT_API_KEY = os.getenv('FIRETEXT_API_KEY') + FIRETEXT_INTERNATIONAL_API_KEY = os.getenv( + 'FIRETEXT_INTERNATIONAL_API_KEY', 'placeholder' + ) # Prefix to identify queues in SQS NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') @@ -137,7 +141,7 @@ class Config(object): ########################### NOTIFY_ENVIRONMENT = 'development' - AWS_REGION = 'eu-west-1' + AWS_REGION = 'us-west-2' INVITATION_EXPIRATION_DAYS = 2 NOTIFY_APP_NAME = 'api' SQLALCHEMY_RECORD_QUERIES = False @@ -160,10 +164,7 @@ class Config(object): CHECK_PROXY_HEADER = False # these should always add up to 100% - SMS_PROVIDER_RESTING_POINTS = { - 'mmg': 50, - 'firetext': 50 - } + SMS_PROVIDER_RESTING_POINTS = {'mmg': 50, 'firetext': 50} NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' @@ -179,10 +180,16 @@ class Config(object): ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573' TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0' TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb' - REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1' + REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = ( + 'a42f1d17-9404-46d5-a647-d013bdfca3e1' + ) MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' - MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' - MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' + MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = ( + 'c20206d5-bf03-4002-9a90-37d5032d9e84' + ) + MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = ( + '522b6657-5ca5-4368-a294-6b527703bd0b' + ) NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' @@ -205,80 +212,85 @@ class Config(object): ], # this is overriden by the -Q command, but locally, we should read from all queues 'task_queues': [ - Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues() + Queue(queue, Exchange('default'), routing_key=queue) + for queue in QueueNames.all_queues() ], 'beat_schedule': { # app/celery/scheduled_tasks.py 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute='0,15,30,45'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'delete-verify-codes': { 'task': 'delete-verify-codes', 'schedule': timedelta(minutes=63), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'delete-invitations': { 'task': 'delete-invitations', 'schedule': timedelta(minutes=66), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'switch-current-sms-provider-on-slow-delivery': { 'task': 'switch-current-sms-provider-on-slow-delivery', 'schedule': crontab(), # Every minute - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-job-status': { 'task': 'check-job-status', 'schedule': crontab(), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'tend-providers-back-to-middle': { 'task': 'tend-providers-back-to-middle', 'schedule': crontab(minute='*/5'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-for-missing-rows-in-completed-jobs': { 'task': 'check-for-missing-rows-in-completed-jobs', 'schedule': crontab(minute='*/10'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'replay-created-notifications': { 'task': 'replay-created-notifications', 'schedule': crontab(minute='0, 15, 30, 45'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, # app/celery/nightly_tasks.py 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(hour=0, minute=5), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'create-nightly-billing': { 'task': 'create-nightly-billing', 'schedule': crontab(hour=0, minute=15), - 'options': {'queue': QueueNames.REPORTING} + 'options': {'queue': QueueNames.REPORTING}, }, 'create-nightly-notification-status': { 'task': 'create-nightly-notification-status', - 'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications' - 'options': {'queue': QueueNames.REPORTING} + 'schedule': crontab( + hour=0, minute=30 + ), # after 'timeout-sending-notifications' + 'options': {'queue': QueueNames.REPORTING}, }, 'delete-notifications-older-than-retention': { 'task': 'delete-notifications-older-than-retention', - 'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' - 'options': {'queue': QueueNames.REPORTING} + 'schedule': crontab( + hour=3, minute=0 + ), # after 'create-nightly-notification-status' + 'options': {'queue': QueueNames.REPORTING}, }, 'delete-inbound-sms': { 'task': 'delete-inbound-sms', 'schedule': crontab(hour=1, minute=40), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'save-daily-notification-processing-time': { 'task': 'save-daily-notification-processing-time', 'schedule': crontab(hour=2, minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'remove_sms_email_jobs': { 'task': 'remove_sms_email_jobs', @@ -294,51 +306,51 @@ class Config(object): 'check-if-letters-still-in-created': { 'task': 'check-if-letters-still-in-created', 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-if-letters-still-pending-virus-check': { 'task': 'check-if-letters-still-pending-virus-check', 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=17, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, # The collate-letter-pdf does assume it is called in an hour that BST does not make a # difference to the truncate date which translates to the filename to process 'collate-letter-pdfs-to-be-sent': { 'task': 'collate-letter-pdfs-to-be-sent', 'schedule': crontab(hour=17, minute=50), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'raise-alert-if-no-letter-ack-file': { 'task': 'raise-alert-if-no-letter-ack-file', 'schedule': crontab(hour=23, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'trigger-link-tests': { 'task': 'trigger-link-tests', 'schedule': timedelta(minutes=15), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'auto-expire-broadcast-messages': { 'task': 'auto-expire-broadcast-messages', 'schedule': timedelta(minutes=5), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'remove-yesterdays-planned-tests-on-govuk-alerts': { 'task': 'remove-yesterdays-planned-tests-on-govuk-alerts', 'schedule': crontab(hour=00, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, - } + }, } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks @@ -364,32 +376,53 @@ class Config(object): FREE_SMS_TIER_FRAGMENT_COUNT = 250000 SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) - FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) + FIRETEXT_INBOUND_SMS_AUTH = json.loads( + os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]') + ) MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) + MMG_INBOUND_SMS_USERNAME = json.loads( + os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]') + ) ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '') ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '') HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) - TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') - TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') + TEMPLATE_PREVIEW_API_HOST = os.environ.get( + 'TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013' + ) + TEMPLATE_PREVIEW_API_KEY = os.environ.get( + 'TEMPLATE_PREVIEW_API_KEY', 'my-secret-key' + ) - DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') - DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') + DOCUMENT_DOWNLOAD_API_HOST = os.environ.get( + 'DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000' + ) + DOCUMENT_DOWNLOAD_API_KEY = os.environ.get( + 'DOCUMENT_DOWNLOAD_API_KEY', 'auth-token' + ) # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` - MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") - FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") - SES_STUB_URL = os.environ.get("SES_STUB_URL") + MMG_URL = os.environ.get('MMG_URL', 'https://api.mmg.co.uk/jsonv2a/api.php') + FIRETEXT_URL = os.environ.get( + 'FIRETEXT_URL', 'https://www.firetext.co.uk/api/sendsms/json' + ) + SES_STUB_URL = os.environ.get('SES_STUB_URL') - AWS_REGION = 'eu-west-1' + AWS_REGION = 'us-west-2' CBC_PROXY_ENABLED = True CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '') - CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '') + CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get( + 'CBC_PROXY_AWS_SECRET_ACCESS_KEY', '' + ) - ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE} + ENABLED_CBCS = { + BroadcastProvider.EE, + BroadcastProvider.THREE, + BroadcastProvider.O2, + BroadcastProvider.VODAFONE, + } # as defined in api db migration 0331_add_broadcast_org.py BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' @@ -399,6 +432,7 @@ class Config(object): # Config overrides ### ###################### + class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False @@ -417,7 +451,7 @@ class Development(Config): INTERNAL_CLIENT_API_KEYS = { Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] + Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'], } SECRET_KEY = 'dev-notify-secret-key' @@ -428,10 +462,13 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' NOTIFY_LOG_PATH = 'application.log' - NOTIFY_EMAIL_DOMAIN = "notify.tools" + NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'dispostable.com') - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) + REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0') ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' @@ -466,17 +503,20 @@ class Test(Development): LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in jenkins and on cloudfoundry - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) CELERY = { **Config.CELERY, - 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' + 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://', } ANTIVIRUS_ENABLED = True API_RATE_LIMIT_ENABLED = True - API_HOST_NAME = "http://localhost:6011" + API_HOST_NAME = 'http://localhost:6011' SMS_INBOUND_WHITELIST = ['203.0.113.195'] FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] @@ -486,7 +526,10 @@ class Test(Development): FIRETEXT_URL = 'https://example.com/firetext' CBC_PROXY_ENABLED = True - DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] + DVLA_EMAIL_ADDRESSES = [ + 'success@simulator.amazonses.com', + 'success+2@simulator.amazonses.com', + ] class Preview(Config): @@ -569,5 +612,5 @@ configs = { 'production': Live, 'staging': Staging, 'preview': Preview, - 'sandbox': Sandbox + 'sandbox': Sandbox, } diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json new file mode 100644 index 000000000..cf46cba59 --- /dev/null +++ b/devcontainer-api/.devcontainer.json @@ -0,0 +1,42 @@ +{ + "name": "notification-api", + "dockerComposeFile": "../docker-compose.devcontainer.yml", + "service": "dev", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", + "remoteEnv": { + "PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence + }, + "settings": { + "[python]": { + "editor.formatOnSave": true + }, + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.pythonPath": "/usr/local/bin/python" + }, + "features": { + "docker-from-docker": { + "version": "latest", + "moby": true + } + }, + "extensions": [ + "ms-python.black-formatter", + "donjayamanne.python-extension-pack", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance", + "eamodio.gitlens", + "wholroyd.jinja", + "pmbenjamin.vscode-snyk", + "visualstudioexptteam.vscodeintellicode", + "yzhang.markdown-all-in-one", + "ms-ossdata.vscode-postgresql", + "GitHub.copilot", + "ms-vsliveshare.vsliveshare", + "mtxr.sqltools", + "mtxr.sqltools-driver-pg", + ], + "postCreateCommand": "notify-dev-entrypoint.sh", + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile new file mode 100644 index 000000000..cdedfcc59 --- /dev/null +++ b/devcontainer-api/Dockerfile @@ -0,0 +1,40 @@ +FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.9 + +RUN apt-get update \ + && apt-get -y install --no-install-recommends \ + apt-utils \ + postgresql-client \ + 2>&1 \ + && apt-get -y install \ + curl \ + emacs \ + exa \ + fd-find \ + git \ + iproute2 \ + less \ + libsodium-dev \ + lsb-release \ + man-db \ + manpages \ + net-tools \ + openssh-client \ + procps \ + sudo \ + tldr \ + unzip \ + vim \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# Upgrade pip +RUN pip install --upgrade pip + +COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/ +COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/ + +ENV SHELL /bin/zsh + +EXPOSE 8000 +EXPOSE 6011 diff --git a/devcontainer-api/initdb/notify-db-entrypoint.sh b/devcontainer-api/initdb/notify-db-entrypoint.sh new file mode 100755 index 000000000..d60f8538a --- /dev/null +++ b/devcontainer-api/initdb/notify-db-entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +# +# The PostgreSQL Docker image has an extension mechanism that does +# not necessitate to override the entrypoint or main command. One +# simply has to copy a shell script into the +# /docker-entrypoint-initdb.d/ initialization folder. +################################################################### + +# Notify database setup. +createdb --user=postgres notification_api + +wall "The db container entrypoint setup is complete!" \ No newline at end of file diff --git a/devcontainer-api/scripts/notify-dev-entrypoint.sh b/devcontainer-api/scripts/notify-dev-entrypoint.sh new file mode 100755 index 000000000..7ae62f240 --- /dev/null +++ b/devcontainer-api/scripts/notify-dev-entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +################################################################### + +# Define aliases +echo -e "\n\n# User's Aliases" >> ~/.zshrc +echo -e "alias fd=fdfind" >> ~/.zshrc +echo -e "alias l='ls -al --color'" >> ~/.zshrc +echo -e "alias ls='exa'" >> ~/.zshrc +echo -e "alias l='exa -alh'" >> ~/.zshrc +echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc +echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc + +# # Kubectl aliases and command autocomplete +# echo -e "alias k='kubectl'" >> ~/.zshrc +# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc +# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc +# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc +# echo -e "complete -F __start_kubectl k" >> ~/.zshrc + +cd /workspace + +# Warm up git index prior to display status in prompt else it will +# be quite slow on every invocation of starship. +git status + +make generate-version-file +pip3 install -r requirements.txt +pip3 install -r requirements_for_test.txt + +# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer +pip3 install virtualenv + +# Upgrade schema of the notification_api database +flask db upgrade + +# Run flask server +# make run-flask diff --git a/devcontainer-api/scripts/notify-worker-entrypoint.sh b/devcontainer-api/scripts/notify-worker-entrypoint.sh new file mode 100755 index 000000000..5ae776548 --- /dev/null +++ b/devcontainer-api/scripts/notify-worker-entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +################################################################### + +# Define aliases +echo -e "\n\n# User's Aliases" >> ~/.zshrc +echo -e "alias fd=fdfind" >> ~/.zshrc +echo -e "alias l='ls -al --color'" >> ~/.zshrc +echo -e "alias ls='exa'" >> ~/.zshrc +echo -e "alias l='exa -alh'" >> ~/.zshrc +echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc +echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc + +# # Kubectl aliases and command autocomplete +# echo -e "alias k='kubectl'" >> ~/.zshrc +# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc +# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc +# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc +# echo -e "complete -F __start_kubectl k" >> ~/.zshrc + +cd /workspace + +# Warm up git index prior to display status in prompt else it will +# be quite slow on every invocation of starship. +git status + +make generate-version-file +pip3 install -r requirements.txt +pip3 install -r requirements_for_test.txt + +# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer +pip3 install virtualenv + +# Update commit hash etc... +make generate-version-file + +# start listening to the queues +# make run-celery diff --git a/docker-compose.devcontainer.yml b/docker-compose.devcontainer.yml new file mode 100644 index 000000000..fd8da349e --- /dev/null +++ b/docker-compose.devcontainer.yml @@ -0,0 +1,72 @@ +version: '3' + +services: + db: + container_name: db + image: postgres:13.4 + volumes: + - ./devcontainer-api/initdb:/docker-entrypoint-initdb.d + environment: + PGGSSENCMODE: disable + POSTGRES_USER: postgres + POSTGRES_PASSWORD: chummy + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - "5432:5432" + expose: + - "5432" + command: + - "postgres" + - "-c" + - "listen_addresses=*" + restart: always + redis: + container_name: redis + image: redis:6.2 + restart: always + command: redis-server --port 6380 + ports: + - "6380:6380" + expose: + - "6380" + dev: + container_name: dev + image: dev-notification-api + build: + context: . + dockerfile: devcontainer-api/Dockerfile + env_file: .env + volumes: + - .:/workspace:cached + command: sleep infinity + ports: + - 6011:6011 + expose: + - "6011" + depends_on: + - db + - redis + links: + - db + restart: always + worker: + container_name: worker + image: dev-notification-api + build: + context: . + dockerfile: devcontainer-api/Dockerfile + env_file: .env + volumes: + - .:/workspace:cached + command: sleep infinity + depends_on: + - db + - redis + - dev + links: + - db + +networks: + default: + name: notify-network + external: true diff --git a/migrations/versions/0025_notify_service_data.py b/migrations/versions/0025_notify_service_data.py index 54abecdd8..52d626479 100644 --- a/migrations/versions/0025_notify_service_data.py +++ b/migrations/versions/0025_notify_service_data.py @@ -25,18 +25,18 @@ def upgrade(): password = hashpw(str(uuid.uuid4())) op.get_bind() user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin) - VALUES ('{}', 'Notify service user', 'notify-service-user@digital.cabinet-office', '{}', 0,'{}', '+441234123412', 'active', False) + VALUES ('{}', 'Notify service user', 'testsender@dispostable.com', '{}', 0,'{}', '+441234123412', 'active', False) """ op.execute(user_insert.format(user_id, datetime.utcnow(), password)) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) - VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', - '{}', 'notify@digital.cabinet-office.gov.uk', 1) + VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com', + '{}', 'testsender@dispostable.com', 1) """ op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id)) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) - VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', - '{}', 'notify@digital.cabinet-office.gov.uk', 1) + VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com', + '{}', 'testsender@dispostable.com', 1) """ op.execute(service_insert.format(service_id, datetime.utcnow(), user_id)) user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')""" diff --git a/migrations/versions/0070_fix_notify_user_email.py b/migrations/versions/0070_fix_notify_user_email.py index fd946df86..fd9f27b3c 100644 --- a/migrations/versions/0070_fix_notify_user_email.py +++ b/migrations/versions/0070_fix_notify_user_email.py @@ -17,7 +17,7 @@ import sqlalchemy as sa def upgrade(): op.execute(""" UPDATE users - SET email_address = 'notify-service-user@digital.cabinet-office.gov.uk' + SET email_address = 'testsender@dispostable.com' WHERE email_address = 'notify-service-user@digital.cabinet-office' """) @@ -26,5 +26,5 @@ def downgrade(): op.execute(""" UPDATE users SET email_address = 'notify-service-user@digital.cabinet-office' - WHERE email_address = 'notify-service-user@digital.cabinet-office.gov.uk' + WHERE email_address = 'testsender@dispostable.com' """) diff --git a/migrations/versions/0082_add_golive_template.py b/migrations/versions/0082_add_golive_template.py index 4b6ab37ec..42fad4bee 100644 --- a/migrations/versions/0082_add_golive_template.py +++ b/migrations/versions/0082_add_golive_template.py @@ -64,7 +64,7 @@ It’s only an emergency if: * a 500 response code appears when you try to send messages using the API If you have one of these emergencies, email details to: -ooh-gov-uk-notify-support@digital.cabinet-office.gov.uk +testsender@dispostable.com ^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team. diff --git a/migrations/versions/0130_service_email_reply_to_row.py b/migrations/versions/0130_service_email_reply_to_row.py index 0adfa2692..6d0a75254 100644 --- a/migrations/versions/0130_service_email_reply_to_row.py +++ b/migrations/versions/0130_service_email_reply_to_row.py @@ -22,7 +22,7 @@ def upgrade(): INSERT INTO service_email_reply_to (id, service_id, email_address, is_default, created_at) VALUES - ('{}','{}', 'notify+1@digital.cabinet-office.gov.uk', 'f', NOW()) + ('{}','{}', 'testsender@dispostable.com', 'f', NOW()) """.format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID)) diff --git a/requirements.in b/requirements.in index 6f65b385c..edb867b3c 100644 --- a/requirements.in +++ b/requirements.in @@ -23,6 +23,7 @@ cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.8.0 Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned +python-dotenv==0.20.0 notifications-python-client==6.3.0 diff --git a/requirements.txt b/requirements.txt index 1def8a2dc..8d4629ca5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,6 +77,8 @@ docopt==0.6.2 # via notifications-python-client docutils==0.16 # via awscli +python-dotenv==0.20.0 + # via -r requirements.in eventlet==0.33.1 # via gunicorn flask==2.1.2 diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 6abdb9e13..313a068ec 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -65,7 +65,7 @@ function start_application { } function start_aws_logs_agent { - exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & + exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & AWSLOGS_AGENT_PID=$! echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index de16f3941..15495b873 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -102,7 +102,7 @@ function start_application { function start_aws_logs_agent { echo "Starting aws logs agent..." - exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & + exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & AWSLOGS_AGENT_PID=$! echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" }