From 59b72f4853c3e9099c12c8289456bd65bf1d4628 Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 13:16:32 -0700 Subject: [PATCH 1/5] add devcontainer configs and docker network orchestration --- .gitignore | 2 + app/celery/research_mode_tasks.py | 20 +-- app/clients/cbc_proxy.py | 2 +- app/config.py | 163 +++++++++++------- devcontainer-api/.devcontainer.json | 42 +++++ devcontainer-api/Dockerfile | 40 +++++ .../initdb/notify-db-entrypoint.sh | 18 ++ .../scripts/notify-dev-entrypoint.sh | 43 +++++ .../scripts/notify-worker-entrypoint.sh | 43 +++++ docker-compose.devcontainer.yml | 72 ++++++++ .../versions/0025_notify_service_data.py | 10 +- .../versions/0070_fix_notify_user_email.py | 4 +- .../versions/0082_add_golive_template.py | 2 +- .../0130_service_email_reply_to_row.py | 2 +- requirements.in | 1 + requirements.txt | 2 + scripts/run_app_paas.sh | 2 +- scripts/run_multi_worker_app_paas.sh | 2 +- 18 files changed, 388 insertions(+), 82 deletions(-) create mode 100644 devcontainer-api/.devcontainer.json create mode 100644 devcontainer-api/Dockerfile create mode 100755 devcontainer-api/initdb/notify-db-entrypoint.sh create mode 100755 devcontainer-api/scripts/notify-dev-entrypoint.sh create mode 100755 devcontainer-api/scripts/notify-worker-entrypoint.sh create mode 100644 docker-compose.devcontainer.yml diff --git a/.gitignore b/.gitignore index 4ae3c89a8..b5c8d94af 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ target/ *.DS_Store environment.sh .envrc +.env +.env* celerybeat-schedule diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index db68a2fe4..e937739f5 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -164,7 +164,7 @@ def ses_notification_callback(reference): 'processingTimeMillis': 2003, 'recipients': ['success@simulator.amazonses.com'], 'remoteMtaIp': '123.123.123.123', - 'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com', + 'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com', 'smtpResponse': '250 2.6.0 Message received', 'timestamp': '2017-11-17T12:14:03.646Z' }, @@ -201,7 +201,7 @@ def ses_notification_callback(reference): 'messageId': reference, 'sendingAccountId': '12341234', 'source': '"TEST" ', - 'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', + 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works', 'sourceIp': '0.0.0.1', 'timestamp': '2017-11-17T12:14:01.643Z' }, @@ -211,14 +211,14 @@ def ses_notification_callback(reference): return { 'Type': 'Notification', 'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a', - 'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', + 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications', 'Subject': None, 'Message': json.dumps(ses_message_body), 'Timestamp': '2017-11-17T12:14:03.710Z', 'SignatureVersion': '1', 'Signature': '[REDACTED]', - 'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem', - 'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]', + 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem', + 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]', 'MessageAttributes': {} } @@ -244,7 +244,7 @@ def _ses_bounce_callback(reference, bounce_type): }], 'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000', 'remoteMtaIp': '123.123.123.123', - 'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com', + 'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com', 'timestamp': '2017-11-17T12:14:05.131Z' }, 'mail': { @@ -280,7 +280,7 @@ def _ses_bounce_callback(reference, bounce_type): 'messageId': reference, 'sendingAccountId': '12341234', 'source': '"TEST" ', - 'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works', + 'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works', 'sourceIp': '0.0.0.1', 'timestamp': '2017-11-17T12:14:03.000Z' }, @@ -289,13 +289,13 @@ def _ses_bounce_callback(reference, bounce_type): return { 'Type': 'Notification', 'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7', - 'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications', + 'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications', 'Subject': None, 'Message': json.dumps(ses_message_body), 'Timestamp': '2017-11-17T12:14:05.149Z', 'SignatureVersion': '1', 'Signature': '[REDACTED]', # noqa - 'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem', - 'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]', + 'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem', + 'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]', 'MessageAttributes': {} } diff --git a/app/clients/cbc_proxy.py b/app/clients/cbc_proxy.py index 89ec65db3..a33b0b284 100644 --- a/app/clients/cbc_proxy.py +++ b/app/clients/cbc_proxy.py @@ -39,7 +39,7 @@ class CBCProxyClient: if app.config.get('CBC_PROXY_ENABLED'): self._lambda_client = boto3.client( 'lambda', - region_name='eu-west-2', + region_name='us-west-2', aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'], aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'], ) diff --git a/app/config.py b/app/config.py index 35bc7a095..fd90ed67e 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,6 @@ import json import os from datetime import timedelta - from celery.schedules import crontab from kombu import Exchange, Queue @@ -100,14 +99,19 @@ class Config(object): DANGEROUS_SALT = os.getenv('DANGEROUS_SALT') # DB conection string - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) # MMG API Key MMG_API_KEY = os.getenv('MMG_API_KEY') # Firetext API Key - FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") - FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") + FIRETEXT_API_KEY = os.getenv('FIRETEXT_API_KEY') + FIRETEXT_INTERNATIONAL_API_KEY = os.getenv( + 'FIRETEXT_INTERNATIONAL_API_KEY', 'placeholder' + ) # Prefix to identify queues in SQS NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') @@ -137,7 +141,7 @@ class Config(object): ########################### NOTIFY_ENVIRONMENT = 'development' - AWS_REGION = 'eu-west-1' + AWS_REGION = 'us-west-2' INVITATION_EXPIRATION_DAYS = 2 NOTIFY_APP_NAME = 'api' SQLALCHEMY_RECORD_QUERIES = False @@ -160,10 +164,7 @@ class Config(object): CHECK_PROXY_HEADER = False # these should always add up to 100% - SMS_PROVIDER_RESTING_POINTS = { - 'mmg': 50, - 'firetext': 50 - } + SMS_PROVIDER_RESTING_POINTS = {'mmg': 50, 'firetext': 50} NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' @@ -179,10 +180,16 @@ class Config(object): ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573' TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0' TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb' - REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1' + REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = ( + 'a42f1d17-9404-46d5-a647-d013bdfca3e1' + ) MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' - MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' - MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' + MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = ( + 'c20206d5-bf03-4002-9a90-37d5032d9e84' + ) + MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = ( + '522b6657-5ca5-4368-a294-6b527703bd0b' + ) NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' @@ -205,80 +212,85 @@ class Config(object): ], # this is overriden by the -Q command, but locally, we should read from all queues 'task_queues': [ - Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues() + Queue(queue, Exchange('default'), routing_key=queue) + for queue in QueueNames.all_queues() ], 'beat_schedule': { # app/celery/scheduled_tasks.py 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute='0,15,30,45'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'delete-verify-codes': { 'task': 'delete-verify-codes', 'schedule': timedelta(minutes=63), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'delete-invitations': { 'task': 'delete-invitations', 'schedule': timedelta(minutes=66), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'switch-current-sms-provider-on-slow-delivery': { 'task': 'switch-current-sms-provider-on-slow-delivery', 'schedule': crontab(), # Every minute - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-job-status': { 'task': 'check-job-status', 'schedule': crontab(), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'tend-providers-back-to-middle': { 'task': 'tend-providers-back-to-middle', 'schedule': crontab(minute='*/5'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-for-missing-rows-in-completed-jobs': { 'task': 'check-for-missing-rows-in-completed-jobs', 'schedule': crontab(minute='*/10'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'replay-created-notifications': { 'task': 'replay-created-notifications', 'schedule': crontab(minute='0, 15, 30, 45'), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, # app/celery/nightly_tasks.py 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(hour=0, minute=5), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'create-nightly-billing': { 'task': 'create-nightly-billing', 'schedule': crontab(hour=0, minute=15), - 'options': {'queue': QueueNames.REPORTING} + 'options': {'queue': QueueNames.REPORTING}, }, 'create-nightly-notification-status': { 'task': 'create-nightly-notification-status', - 'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications' - 'options': {'queue': QueueNames.REPORTING} + 'schedule': crontab( + hour=0, minute=30 + ), # after 'timeout-sending-notifications' + 'options': {'queue': QueueNames.REPORTING}, }, 'delete-notifications-older-than-retention': { 'task': 'delete-notifications-older-than-retention', - 'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' - 'options': {'queue': QueueNames.REPORTING} + 'schedule': crontab( + hour=3, minute=0 + ), # after 'create-nightly-notification-status' + 'options': {'queue': QueueNames.REPORTING}, }, 'delete-inbound-sms': { 'task': 'delete-inbound-sms', 'schedule': crontab(hour=1, minute=40), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'save-daily-notification-processing-time': { 'task': 'save-daily-notification-processing-time', 'schedule': crontab(hour=2, minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'remove_sms_email_jobs': { 'task': 'remove_sms_email_jobs', @@ -294,51 +306,51 @@ class Config(object): 'check-if-letters-still-in-created': { 'task': 'check-if-letters-still-in-created', 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-if-letters-still-pending-virus-check': { 'task': 'check-if-letters-still-pending-virus-check', 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=17, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, # The collate-letter-pdf does assume it is called in an hour that BST does not make a # difference to the truncate date which translates to the filename to process 'collate-letter-pdfs-to-be-sent': { 'task': 'collate-letter-pdfs-to-be-sent', 'schedule': crontab(hour=17, minute=50), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'raise-alert-if-no-letter-ack-file': { 'task': 'raise-alert-if-no-letter-ack-file', 'schedule': crontab(hour=23, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'trigger-link-tests': { 'task': 'trigger-link-tests', 'schedule': timedelta(minutes=15), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'auto-expire-broadcast-messages': { 'task': 'auto-expire-broadcast-messages', 'schedule': timedelta(minutes=5), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, 'remove-yesterdays-planned-tests-on-govuk-alerts': { 'task': 'remove-yesterdays-planned-tests-on-govuk-alerts', 'schedule': crontab(hour=00, minute=00), - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.PERIODIC}, }, - } + }, } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks @@ -364,32 +376,53 @@ class Config(object): FREE_SMS_TIER_FRAGMENT_COUNT = 250000 SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) - FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) + FIRETEXT_INBOUND_SMS_AUTH = json.loads( + os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]') + ) MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) + MMG_INBOUND_SMS_USERNAME = json.loads( + os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]') + ) ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '') ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '') HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) - TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') - TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') + TEMPLATE_PREVIEW_API_HOST = os.environ.get( + 'TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013' + ) + TEMPLATE_PREVIEW_API_KEY = os.environ.get( + 'TEMPLATE_PREVIEW_API_KEY', 'my-secret-key' + ) - DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') - DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') + DOCUMENT_DOWNLOAD_API_HOST = os.environ.get( + 'DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000' + ) + DOCUMENT_DOWNLOAD_API_KEY = os.environ.get( + 'DOCUMENT_DOWNLOAD_API_KEY', 'auth-token' + ) # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` - MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") - FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") - SES_STUB_URL = os.environ.get("SES_STUB_URL") + MMG_URL = os.environ.get('MMG_URL', 'https://api.mmg.co.uk/jsonv2a/api.php') + FIRETEXT_URL = os.environ.get( + 'FIRETEXT_URL', 'https://www.firetext.co.uk/api/sendsms/json' + ) + SES_STUB_URL = os.environ.get('SES_STUB_URL') - AWS_REGION = 'eu-west-1' + AWS_REGION = 'us-west-2' CBC_PROXY_ENABLED = True CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '') - CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '') + CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get( + 'CBC_PROXY_AWS_SECRET_ACCESS_KEY', '' + ) - ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE} + ENABLED_CBCS = { + BroadcastProvider.EE, + BroadcastProvider.THREE, + BroadcastProvider.O2, + BroadcastProvider.VODAFONE, + } # as defined in api db migration 0331_add_broadcast_org.py BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' @@ -399,6 +432,7 @@ class Config(object): # Config overrides ### ###################### + class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False @@ -417,7 +451,7 @@ class Development(Config): INTERNAL_CLIENT_API_KEYS = { Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] + Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'], } SECRET_KEY = 'dev-notify-secret-key' @@ -428,10 +462,13 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' NOTIFY_LOG_PATH = 'application.log' - NOTIFY_EMAIL_DOMAIN = "notify.tools" + NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'dispostable.com') - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api') - REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) + REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0') ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' @@ -466,17 +503,20 @@ class Test(Development): LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in jenkins and on cloudfoundry - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api', + ) CELERY = { **Config.CELERY, - 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' + 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://', } ANTIVIRUS_ENABLED = True API_RATE_LIMIT_ENABLED = True - API_HOST_NAME = "http://localhost:6011" + API_HOST_NAME = 'http://localhost:6011' SMS_INBOUND_WHITELIST = ['203.0.113.195'] FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] @@ -486,7 +526,10 @@ class Test(Development): FIRETEXT_URL = 'https://example.com/firetext' CBC_PROXY_ENABLED = True - DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] + DVLA_EMAIL_ADDRESSES = [ + 'success@simulator.amazonses.com', + 'success+2@simulator.amazonses.com', + ] class Preview(Config): @@ -569,5 +612,5 @@ configs = { 'production': Live, 'staging': Staging, 'preview': Preview, - 'sandbox': Sandbox + 'sandbox': Sandbox, } diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json new file mode 100644 index 000000000..cf46cba59 --- /dev/null +++ b/devcontainer-api/.devcontainer.json @@ -0,0 +1,42 @@ +{ + "name": "notification-api", + "dockerComposeFile": "../docker-compose.devcontainer.yml", + "service": "dev", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", + "remoteEnv": { + "PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence + }, + "settings": { + "[python]": { + "editor.formatOnSave": true + }, + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.pythonPath": "/usr/local/bin/python" + }, + "features": { + "docker-from-docker": { + "version": "latest", + "moby": true + } + }, + "extensions": [ + "ms-python.black-formatter", + "donjayamanne.python-extension-pack", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance", + "eamodio.gitlens", + "wholroyd.jinja", + "pmbenjamin.vscode-snyk", + "visualstudioexptteam.vscodeintellicode", + "yzhang.markdown-all-in-one", + "ms-ossdata.vscode-postgresql", + "GitHub.copilot", + "ms-vsliveshare.vsliveshare", + "mtxr.sqltools", + "mtxr.sqltools-driver-pg", + ], + "postCreateCommand": "notify-dev-entrypoint.sh", + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile new file mode 100644 index 000000000..cdedfcc59 --- /dev/null +++ b/devcontainer-api/Dockerfile @@ -0,0 +1,40 @@ +FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.9 + +RUN apt-get update \ + && apt-get -y install --no-install-recommends \ + apt-utils \ + postgresql-client \ + 2>&1 \ + && apt-get -y install \ + curl \ + emacs \ + exa \ + fd-find \ + git \ + iproute2 \ + less \ + libsodium-dev \ + lsb-release \ + man-db \ + manpages \ + net-tools \ + openssh-client \ + procps \ + sudo \ + tldr \ + unzip \ + vim \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +# Upgrade pip +RUN pip install --upgrade pip + +COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/ +COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/ + +ENV SHELL /bin/zsh + +EXPOSE 8000 +EXPOSE 6011 diff --git a/devcontainer-api/initdb/notify-db-entrypoint.sh b/devcontainer-api/initdb/notify-db-entrypoint.sh new file mode 100755 index 000000000..d60f8538a --- /dev/null +++ b/devcontainer-api/initdb/notify-db-entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +# +# The PostgreSQL Docker image has an extension mechanism that does +# not necessitate to override the entrypoint or main command. One +# simply has to copy a shell script into the +# /docker-entrypoint-initdb.d/ initialization folder. +################################################################### + +# Notify database setup. +createdb --user=postgres notification_api + +wall "The db container entrypoint setup is complete!" \ No newline at end of file diff --git a/devcontainer-api/scripts/notify-dev-entrypoint.sh b/devcontainer-api/scripts/notify-dev-entrypoint.sh new file mode 100755 index 000000000..7ae62f240 --- /dev/null +++ b/devcontainer-api/scripts/notify-dev-entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +################################################################### + +# Define aliases +echo -e "\n\n# User's Aliases" >> ~/.zshrc +echo -e "alias fd=fdfind" >> ~/.zshrc +echo -e "alias l='ls -al --color'" >> ~/.zshrc +echo -e "alias ls='exa'" >> ~/.zshrc +echo -e "alias l='exa -alh'" >> ~/.zshrc +echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc +echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc + +# # Kubectl aliases and command autocomplete +# echo -e "alias k='kubectl'" >> ~/.zshrc +# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc +# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc +# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc +# echo -e "complete -F __start_kubectl k" >> ~/.zshrc + +cd /workspace + +# Warm up git index prior to display status in prompt else it will +# be quite slow on every invocation of starship. +git status + +make generate-version-file +pip3 install -r requirements.txt +pip3 install -r requirements_for_test.txt + +# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer +pip3 install virtualenv + +# Upgrade schema of the notification_api database +flask db upgrade + +# Run flask server +# make run-flask diff --git a/devcontainer-api/scripts/notify-worker-entrypoint.sh b/devcontainer-api/scripts/notify-worker-entrypoint.sh new file mode 100755 index 000000000..5ae776548 --- /dev/null +++ b/devcontainer-api/scripts/notify-worker-entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +################################################################### + +# Define aliases +echo -e "\n\n# User's Aliases" >> ~/.zshrc +echo -e "alias fd=fdfind" >> ~/.zshrc +echo -e "alias l='ls -al --color'" >> ~/.zshrc +echo -e "alias ls='exa'" >> ~/.zshrc +echo -e "alias l='exa -alh'" >> ~/.zshrc +echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc +echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc + +# # Kubectl aliases and command autocomplete +# echo -e "alias k='kubectl'" >> ~/.zshrc +# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc +# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc +# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc +# echo -e "complete -F __start_kubectl k" >> ~/.zshrc + +cd /workspace + +# Warm up git index prior to display status in prompt else it will +# be quite slow on every invocation of starship. +git status + +make generate-version-file +pip3 install -r requirements.txt +pip3 install -r requirements_for_test.txt + +# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer +pip3 install virtualenv + +# Update commit hash etc... +make generate-version-file + +# start listening to the queues +# make run-celery diff --git a/docker-compose.devcontainer.yml b/docker-compose.devcontainer.yml new file mode 100644 index 000000000..fd8da349e --- /dev/null +++ b/docker-compose.devcontainer.yml @@ -0,0 +1,72 @@ +version: '3' + +services: + db: + container_name: db + image: postgres:13.4 + volumes: + - ./devcontainer-api/initdb:/docker-entrypoint-initdb.d + environment: + PGGSSENCMODE: disable + POSTGRES_USER: postgres + POSTGRES_PASSWORD: chummy + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - "5432:5432" + expose: + - "5432" + command: + - "postgres" + - "-c" + - "listen_addresses=*" + restart: always + redis: + container_name: redis + image: redis:6.2 + restart: always + command: redis-server --port 6380 + ports: + - "6380:6380" + expose: + - "6380" + dev: + container_name: dev + image: dev-notification-api + build: + context: . + dockerfile: devcontainer-api/Dockerfile + env_file: .env + volumes: + - .:/workspace:cached + command: sleep infinity + ports: + - 6011:6011 + expose: + - "6011" + depends_on: + - db + - redis + links: + - db + restart: always + worker: + container_name: worker + image: dev-notification-api + build: + context: . + dockerfile: devcontainer-api/Dockerfile + env_file: .env + volumes: + - .:/workspace:cached + command: sleep infinity + depends_on: + - db + - redis + - dev + links: + - db + +networks: + default: + name: notify-network + external: true diff --git a/migrations/versions/0025_notify_service_data.py b/migrations/versions/0025_notify_service_data.py index 54abecdd8..52d626479 100644 --- a/migrations/versions/0025_notify_service_data.py +++ b/migrations/versions/0025_notify_service_data.py @@ -25,18 +25,18 @@ def upgrade(): password = hashpw(str(uuid.uuid4())) op.get_bind() user_insert = """INSERT INTO users (id, name, email_address, created_at, failed_login_count, _password, mobile_number, state, platform_admin) - VALUES ('{}', 'Notify service user', 'notify-service-user@digital.cabinet-office', '{}', 0,'{}', '+441234123412', 'active', False) + VALUES ('{}', 'Notify service user', 'testsender@dispostable.com', '{}', 0,'{}', '+441234123412', 'active', False) """ op.execute(user_insert.format(user_id, datetime.utcnow(), password)) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) - VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', - '{}', 'notify@digital.cabinet-office.gov.uk', 1) + VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com', + '{}', 'testsender@dispostable.com', 1) """ op.execute(service_history_insert.format(service_id, datetime.utcnow(), user_id)) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) - VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'notify@digital.cabinet-office.gov.uk', - '{}', 'notify@digital.cabinet-office.gov.uk', 1) + VALUES ('{}', 'Notify service', '{}', True, 1000, False, False, 'testsender@dispostable.com', + '{}', 'testsender@dispostable.com', 1) """ op.execute(service_insert.format(service_id, datetime.utcnow(), user_id)) user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES ('{}', '{}')""" diff --git a/migrations/versions/0070_fix_notify_user_email.py b/migrations/versions/0070_fix_notify_user_email.py index fd946df86..fd9f27b3c 100644 --- a/migrations/versions/0070_fix_notify_user_email.py +++ b/migrations/versions/0070_fix_notify_user_email.py @@ -17,7 +17,7 @@ import sqlalchemy as sa def upgrade(): op.execute(""" UPDATE users - SET email_address = 'notify-service-user@digital.cabinet-office.gov.uk' + SET email_address = 'testsender@dispostable.com' WHERE email_address = 'notify-service-user@digital.cabinet-office' """) @@ -26,5 +26,5 @@ def downgrade(): op.execute(""" UPDATE users SET email_address = 'notify-service-user@digital.cabinet-office' - WHERE email_address = 'notify-service-user@digital.cabinet-office.gov.uk' + WHERE email_address = 'testsender@dispostable.com' """) diff --git a/migrations/versions/0082_add_golive_template.py b/migrations/versions/0082_add_golive_template.py index 4b6ab37ec..42fad4bee 100644 --- a/migrations/versions/0082_add_golive_template.py +++ b/migrations/versions/0082_add_golive_template.py @@ -64,7 +64,7 @@ It’s only an emergency if: * a 500 response code appears when you try to send messages using the API If you have one of these emergencies, email details to: -ooh-gov-uk-notify-support@digital.cabinet-office.gov.uk +testsender@dispostable.com ^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team. diff --git a/migrations/versions/0130_service_email_reply_to_row.py b/migrations/versions/0130_service_email_reply_to_row.py index 0adfa2692..6d0a75254 100644 --- a/migrations/versions/0130_service_email_reply_to_row.py +++ b/migrations/versions/0130_service_email_reply_to_row.py @@ -22,7 +22,7 @@ def upgrade(): INSERT INTO service_email_reply_to (id, service_id, email_address, is_default, created_at) VALUES - ('{}','{}', 'notify+1@digital.cabinet-office.gov.uk', 'f', NOW()) + ('{}','{}', 'testsender@dispostable.com', 'f', NOW()) """.format(EMAIL_REPLY_TO_ID, NOTIFY_SERVICE_ID)) diff --git a/requirements.in b/requirements.in index 6f65b385c..edb867b3c 100644 --- a/requirements.in +++ b/requirements.in @@ -23,6 +23,7 @@ cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.8.0 Werkzeug==2.0.3 # pyup: <2.1.0 # later versions are not compatible with the version of flask-sqlalchemy we have pinned +python-dotenv==0.20.0 notifications-python-client==6.3.0 diff --git a/requirements.txt b/requirements.txt index 1def8a2dc..8d4629ca5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,6 +77,8 @@ docopt==0.6.2 # via notifications-python-client docutils==0.16 # via awscli +python-dotenv==0.20.0 + # via -r requirements.in eventlet==0.33.1 # via gunicorn flask==2.1.2 diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 6abdb9e13..313a068ec 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -65,7 +65,7 @@ function start_application { } function start_aws_logs_agent { - exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & + exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & AWSLOGS_AGENT_PID=$! echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index de16f3941..15495b873 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -102,7 +102,7 @@ function start_application { function start_aws_logs_agent { echo "Starting aws logs agent..." - exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & + exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & AWSLOGS_AGENT_PID=$! echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } From 60262d60317ff45462d6170ce0f7943bb2b043ed Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 13:45:07 -0700 Subject: [PATCH 2/5] config formatting --- Makefile | 15 ++--- app/config.py | 159 ++++++++++++++++++-------------------------------- 2 files changed, 66 insertions(+), 108 deletions(-) diff --git a/Makefile b/Makefile index 2a8a6ac7c..c1aaeb999 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials bootstrap: generate-version-file ## Set up everything to run the app pip3 install -r requirements_for_test.txt createdb notification_api || true - (. environment.sh && flask db upgrade) || true + (flask db upgrade) || true .PHONY: bootstrap-with-docker bootstrap-with-docker: ## Build the image to run the app in Docker @@ -33,11 +33,12 @@ bootstrap-with-docker: ## Build the image to run the app in Docker .PHONY: run-flask run-flask: ## Run flask - . environment.sh && flask run -p 6011 + flask run -p 6011 --host=0.0.0.0 .PHONY: run-celery -run-celery: ## Run celery - . environment.sh && celery \ +run-celery: ## Run celery, TODO remove purge for staging/prod + celery -A run_celery.notify_celery purge -f + celery \ -A run_celery.notify_celery worker \ --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ @@ -49,9 +50,9 @@ run-celery-with-docker: ## Run celery in Docker container (useful if you can't i .PHONY: run-celery-beat run-celery-beat: ## Run celery beat - . environment.sh && celery \ - -A run_celery.notify_celery beat \ - --loglevel=INFO + celery \ + -A run_celery.notify_celery beat \ + --loglevel=INFO .PHONY: run-celery-beat-with-docker run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally) diff --git a/app/config.py b/app/config.py index fd90ed67e..30a6e49de 100644 --- a/app/config.py +++ b/app/config.py @@ -1,6 +1,7 @@ import json import os from datetime import timedelta + from celery.schedules import crontab from kombu import Exchange, Queue @@ -99,19 +100,14 @@ class Config(object): DANGEROUS_SALT = os.getenv('DANGEROUS_SALT') # DB conection string - SQLALCHEMY_DATABASE_URI = os.getenv( - 'SQLALCHEMY_DATABASE_URI', - 'postgresql://postgres:chummy@db:5432/notification_api', - ) + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') # MMG API Key MMG_API_KEY = os.getenv('MMG_API_KEY') # Firetext API Key - FIRETEXT_API_KEY = os.getenv('FIRETEXT_API_KEY') - FIRETEXT_INTERNATIONAL_API_KEY = os.getenv( - 'FIRETEXT_INTERNATIONAL_API_KEY', 'placeholder' - ) + FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY") + FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # Prefix to identify queues in SQS NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX') @@ -164,7 +160,10 @@ class Config(object): CHECK_PROXY_HEADER = False # these should always add up to 100% - SMS_PROVIDER_RESTING_POINTS = {'mmg': 50, 'firetext': 50} + SMS_PROVIDER_RESTING_POINTS = { + 'mmg': 50, + 'firetext': 50 + } NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' @@ -180,16 +179,10 @@ class Config(object): ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573' TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0' TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb' - REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = ( - 'a42f1d17-9404-46d5-a647-d013bdfca3e1' - ) + REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1' MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' - MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = ( - 'c20206d5-bf03-4002-9a90-37d5032d9e84' - ) - MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = ( - '522b6657-5ca5-4368-a294-6b527703bd0b' - ) + MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' + MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' @@ -212,85 +205,80 @@ class Config(object): ], # this is overriden by the -Q command, but locally, we should read from all queues 'task_queues': [ - Queue(queue, Exchange('default'), routing_key=queue) - for queue in QueueNames.all_queues() + Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues() ], 'beat_schedule': { # app/celery/scheduled_tasks.py 'run-scheduled-jobs': { 'task': 'run-scheduled-jobs', 'schedule': crontab(minute='0,15,30,45'), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-verify-codes': { 'task': 'delete-verify-codes', 'schedule': timedelta(minutes=63), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'delete-invitations': { 'task': 'delete-invitations', 'schedule': timedelta(minutes=66), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'switch-current-sms-provider-on-slow-delivery': { 'task': 'switch-current-sms-provider-on-slow-delivery', 'schedule': crontab(), # Every minute - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'check-job-status': { 'task': 'check-job-status', 'schedule': crontab(), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'tend-providers-back-to-middle': { 'task': 'tend-providers-back-to-middle', 'schedule': crontab(minute='*/5'), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'check-for-missing-rows-in-completed-jobs': { 'task': 'check-for-missing-rows-in-completed-jobs', 'schedule': crontab(minute='*/10'), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'replay-created-notifications': { 'task': 'replay-created-notifications', 'schedule': crontab(minute='0, 15, 30, 45'), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, # app/celery/nightly_tasks.py 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(hour=0, minute=5), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'create-nightly-billing': { 'task': 'create-nightly-billing', 'schedule': crontab(hour=0, minute=15), - 'options': {'queue': QueueNames.REPORTING}, + 'options': {'queue': QueueNames.REPORTING} }, 'create-nightly-notification-status': { 'task': 'create-nightly-notification-status', - 'schedule': crontab( - hour=0, minute=30 - ), # after 'timeout-sending-notifications' - 'options': {'queue': QueueNames.REPORTING}, + 'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications' + 'options': {'queue': QueueNames.REPORTING} }, 'delete-notifications-older-than-retention': { 'task': 'delete-notifications-older-than-retention', - 'schedule': crontab( - hour=3, minute=0 - ), # after 'create-nightly-notification-status' - 'options': {'queue': QueueNames.REPORTING}, + 'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' + 'options': {'queue': QueueNames.REPORTING} }, 'delete-inbound-sms': { 'task': 'delete-inbound-sms', 'schedule': crontab(hour=1, minute=40), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'save-daily-notification-processing-time': { 'task': 'save-daily-notification-processing-time', 'schedule': crontab(hour=2, minute=0), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'remove_sms_email_jobs': { 'task': 'remove_sms_email_jobs', @@ -306,51 +294,51 @@ class Config(object): 'check-if-letters-still-in-created': { 'task': 'check-if-letters-still-in-created', 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'check-if-letters-still-pending-virus-check': { 'task': 'check-if-letters-still-pending-virus-check', 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=17, minute=00), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, # The collate-letter-pdf does assume it is called in an hour that BST does not make a # difference to the truncate date which translates to the filename to process 'collate-letter-pdfs-to-be-sent': { 'task': 'collate-letter-pdfs-to-be-sent', 'schedule': crontab(hour=17, minute=50), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'raise-alert-if-no-letter-ack-file': { 'task': 'raise-alert-if-no-letter-ack-file', 'schedule': crontab(hour=23, minute=00), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'trigger-link-tests': { 'task': 'trigger-link-tests', 'schedule': timedelta(minutes=15), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'auto-expire-broadcast-messages': { 'task': 'auto-expire-broadcast-messages', 'schedule': timedelta(minutes=5), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, 'remove-yesterdays-planned-tests-on-govuk-alerts': { 'task': 'remove-yesterdays-planned-tests-on-govuk-alerts', 'schedule': crontab(hour=00, minute=00), - 'options': {'queue': QueueNames.PERIODIC}, + 'options': {'queue': QueueNames.PERIODIC} }, - }, + } } # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks @@ -376,53 +364,32 @@ class Config(object): FREE_SMS_TIER_FRAGMENT_COUNT = 250000 SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) - FIRETEXT_INBOUND_SMS_AUTH = json.loads( - os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]') - ) + FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_USERNAME = json.loads( - os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]') - ) + MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '') ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '') HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) - TEMPLATE_PREVIEW_API_HOST = os.environ.get( - 'TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013' - ) - TEMPLATE_PREVIEW_API_KEY = os.environ.get( - 'TEMPLATE_PREVIEW_API_KEY', 'my-secret-key' - ) + TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') + TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') - DOCUMENT_DOWNLOAD_API_HOST = os.environ.get( - 'DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000' - ) - DOCUMENT_DOWNLOAD_API_KEY = os.environ.get( - 'DOCUMENT_DOWNLOAD_API_KEY', 'auth-token' - ) + DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') + DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` - MMG_URL = os.environ.get('MMG_URL', 'https://api.mmg.co.uk/jsonv2a/api.php') - FIRETEXT_URL = os.environ.get( - 'FIRETEXT_URL', 'https://www.firetext.co.uk/api/sendsms/json' - ) - SES_STUB_URL = os.environ.get('SES_STUB_URL') + MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") + FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") + SES_STUB_URL = os.environ.get("SES_STUB_URL") AWS_REGION = 'us-west-2' CBC_PROXY_ENABLED = True CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '') - CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get( - 'CBC_PROXY_AWS_SECRET_ACCESS_KEY', '' - ) + CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '') - ENABLED_CBCS = { - BroadcastProvider.EE, - BroadcastProvider.THREE, - BroadcastProvider.O2, - BroadcastProvider.VODAFONE, - } + ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE} # as defined in api db migration 0331_add_broadcast_org.py BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' @@ -432,7 +399,6 @@ class Config(object): # Config overrides ### ###################### - class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False @@ -451,7 +417,7 @@ class Development(Config): INTERNAL_CLIENT_API_KEYS = { Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'], + Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] } SECRET_KEY = 'dev-notify-secret-key' @@ -462,13 +428,10 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' NOTIFY_LOG_PATH = 'application.log' - NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'dispostable.com') + NOTIFY_EMAIL_DOMAIN = "dispostable.com" - SQLALCHEMY_DATABASE_URI = os.getenv( - 'SQLALCHEMY_DATABASE_URI', - 'postgresql://postgres:chummy@db:5432/notification_api', - ) - REDIS_URL = os.getenv('REDIS_URL', 'redis://redis:6379/0') + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1' @@ -503,20 +466,17 @@ class Test(Development): LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in jenkins and on cloudfoundry - SQLALCHEMY_DATABASE_URI = os.getenv( - 'SQLALCHEMY_DATABASE_URI', - 'postgresql://postgres:chummy@db:5432/notification_api', - ) + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') CELERY = { **Config.CELERY, - 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://', + 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' } ANTIVIRUS_ENABLED = True API_RATE_LIMIT_ENABLED = True - API_HOST_NAME = 'http://localhost:6011' + API_HOST_NAME = "http://localhost:6011" SMS_INBOUND_WHITELIST = ['203.0.113.195'] FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] @@ -526,10 +486,7 @@ class Test(Development): FIRETEXT_URL = 'https://example.com/firetext' CBC_PROXY_ENABLED = True - DVLA_EMAIL_ADDRESSES = [ - 'success@simulator.amazonses.com', - 'success+2@simulator.amazonses.com', - ] + DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] class Preview(Config): @@ -612,5 +569,5 @@ configs = { 'production': Live, 'staging': Staging, 'preview': Preview, - 'sandbox': Sandbox, + 'sandbox': Sandbox } From f5beb23a1debf2ed27da06bcd611d45016122d21 Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 13:48:30 -0700 Subject: [PATCH 3/5] rm deprecated config --- devcontainer-api/Dockerfile | 1 - .../scripts/notify-worker-entrypoint.sh | 43 ------------------- 2 files changed, 44 deletions(-) delete mode 100755 devcontainer-api/scripts/notify-worker-entrypoint.sh diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile index cdedfcc59..b67a131a8 100644 --- a/devcontainer-api/Dockerfile +++ b/devcontainer-api/Dockerfile @@ -32,7 +32,6 @@ RUN apt-get update \ RUN pip install --upgrade pip COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/ -COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/ ENV SHELL /bin/zsh diff --git a/devcontainer-api/scripts/notify-worker-entrypoint.sh b/devcontainer-api/scripts/notify-worker-entrypoint.sh deleted file mode 100755 index 5ae776548..000000000 --- a/devcontainer-api/scripts/notify-worker-entrypoint.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -ex - -################################################################### -# This script will get executed *once* the Docker container has -# been built. Commands that need to be executed with all available -# tools and the filesystem mount enabled should be located here. -################################################################### - -# Define aliases -echo -e "\n\n# User's Aliases" >> ~/.zshrc -echo -e "alias fd=fdfind" >> ~/.zshrc -echo -e "alias l='ls -al --color'" >> ~/.zshrc -echo -e "alias ls='exa'" >> ~/.zshrc -echo -e "alias l='exa -alh'" >> ~/.zshrc -echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc -echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc - -# # Kubectl aliases and command autocomplete -# echo -e "alias k='kubectl'" >> ~/.zshrc -# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc -# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc -# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc -# echo -e "complete -F __start_kubectl k" >> ~/.zshrc - -cd /workspace - -# Warm up git index prior to display status in prompt else it will -# be quite slow on every invocation of starship. -git status - -make generate-version-file -pip3 install -r requirements.txt -pip3 install -r requirements_for_test.txt - -# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer -pip3 install virtualenv - -# Update commit hash etc... -make generate-version-file - -# start listening to the queues -# make run-celery From e88e36712ff502421274d8d54611532eacc2eb45 Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 14:50:22 -0700 Subject: [PATCH 4/5] sample env --- sample.env | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 sample.env diff --git a/sample.env b/sample.env new file mode 100644 index 000000000..3edac2c06 --- /dev/null +++ b/sample.env @@ -0,0 +1,39 @@ +# Debug +DEBUG=True +ANTIVIRUS_ENABLED=0 +NOTIFY_ENVIRONMENT=development +NOTIFICATION_QUEUE_PREFIX=local_dev_10x +STATSD_HOST=localhost +SES_STUB_URL=None +NOTIFY_APP_NAME=api +NOTIFY_EMAIL_DOMAIN=dispostable.com + +# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API +ADMIN_CLIENT_ID=notify-admin +GOVUK_ALERTS_CLIENT_ID=govuk-alerts + +# Flask +FLASK_APP=application.py +FLASK_ENV=development +WERKZEUG_DEBUG_PIN=off +SECRET_KEY=dev-notify-secret-key +DANGEROUS_SALT=dev-notify-salt + +# URL of admin app +ADMIN_BASE_URL=http://admin:6012 + +# URL of api app (on AWS this is the internal api endpoint) +API_HOST_NAME=http://dev:6011 + +# URL of redis instance +REDIS_URL=redis://redis:6380 +REDIS_ENABLED=1 + +# DB conection string, this is overriden in jenkins and on cloudfoundry +SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api +SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api + +# AWS +AWS_REGION=us-west-2 +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= From 79ba6cc1d12670582528563d35edef25f93017fb Mon Sep 17 00:00:00 2001 From: Jim Moffet Date: Mon, 13 Jun 2022 21:42:36 -0700 Subject: [PATCH 5/5] disable cache persistence & env updates --- app/config.py | 3 +- app/models.py | 4 +- devcontainer-api/Dockerfile | 1 + .../scripts/notify-worker-entrypoint.sh | 40 ++++++++++++++++++ devcontainer-worker/.devcontainer.json | 42 +++++++++++++++++++ docker-compose.devcontainer.yml | 2 +- migrations/versions/0029_fix_email_from.py | 4 +- sample.env | 4 +- 8 files changed, 92 insertions(+), 8 deletions(-) create mode 100755 devcontainer-api/scripts/notify-worker-entrypoint.sh create mode 100644 devcontainer-worker/.devcontainer.json diff --git a/app/config.py b/app/config.py index 30a6e49de..9381ae6cc 100644 --- a/app/config.py +++ b/app/config.py @@ -152,6 +152,8 @@ class Config(object): ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 5 MAX_FAILED_LOGIN_COUNT = 10 + + SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # be careful increasing this size without being sure that we won't see slowness in pysftp MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb @@ -381,7 +383,6 @@ class Config(object): # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") - SES_STUB_URL = os.environ.get("SES_STUB_URL") AWS_REGION = 'us-west-2' diff --git a/app/models.py b/app/models.py index cbdc1779f..a0485e0cd 100644 --- a/app/models.py +++ b/app/models.py @@ -122,7 +122,7 @@ class User(db.Model): state = db.Column(db.String, nullable=False, default='pending') platform_admin = db.Column(db.Boolean, nullable=False, default=False) current_session_id = db.Column(UUID(as_uuid=True), nullable=True) - auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=SMS_AUTH_TYPE) + auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE) email_access_validated_at = db.Column( db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow ) @@ -1841,7 +1841,7 @@ class InvitedUser(db.Model): db.ForeignKey('auth_type.name'), index=True, nullable=False, - default=SMS_AUTH_TYPE + default=EMAIL_AUTH_TYPE ) folder_permissions = db.Column(JSONB(none_as_null=True), nullable=False, default=[]) diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile index b67a131a8..cdedfcc59 100644 --- a/devcontainer-api/Dockerfile +++ b/devcontainer-api/Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update \ RUN pip install --upgrade pip COPY devcontainer-api/scripts/notify-dev-entrypoint.sh /usr/local/bin/ +COPY devcontainer-api/scripts/notify-worker-entrypoint.sh /usr/local/bin/ ENV SHELL /bin/zsh diff --git a/devcontainer-api/scripts/notify-worker-entrypoint.sh b/devcontainer-api/scripts/notify-worker-entrypoint.sh new file mode 100755 index 000000000..98251ef15 --- /dev/null +++ b/devcontainer-api/scripts/notify-worker-entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -ex + +################################################################### +# This script will get executed *once* the Docker container has +# been built. Commands that need to be executed with all available +# tools and the filesystem mount enabled should be located here. +################################################################### + +# Define aliases +echo -e "\n\n# User's Aliases" >> ~/.zshrc +echo -e "alias fd=fdfind" >> ~/.zshrc +echo -e "alias l='ls -al --color'" >> ~/.zshrc +echo -e "alias ls='exa'" >> ~/.zshrc +echo -e "alias l='exa -alh'" >> ~/.zshrc +echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc +echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc + +# # Kubectl aliases and command autocomplete +# echo -e "alias k='kubectl'" >> ~/.zshrc +# echo -e "alias k-staging='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-staging-eks-cluster'" >> ~/.zshrc +# echo -e "alias k-prod='aws eks --region ca-central-1 update-kubeconfig --name notification-canada-ca-production-eks-cluster'" >> ~/.zshrc +# echo -e "source <(kubectl completion zsh)" >> ~/.zshrc +# echo -e "complete -F __start_kubectl k" >> ~/.zshrc + +cd /workspace + +# Warm up git index prior to display status in prompt else it will +# be quite slow on every invocation of starship. +git status + +make generate-version-file +pip3 install -r requirements.txt +pip3 install -r requirements_for_test.txt + +# Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer +pip3 install virtualenv + +# Run Celery worker +# make run-celery diff --git a/devcontainer-worker/.devcontainer.json b/devcontainer-worker/.devcontainer.json new file mode 100644 index 000000000..c044e8896 --- /dev/null +++ b/devcontainer-worker/.devcontainer.json @@ -0,0 +1,42 @@ +{ + "name": "notification-api", + "dockerComposeFile": "../docker-compose.devcontainer.yml", + "service": "worker", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", + "remoteEnv": { + "PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence + }, + "settings": { + "[python]": { + "editor.formatOnSave": true + }, + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.pythonPath": "/usr/local/bin/python" + }, + "features": { + "docker-from-docker": { + "version": "latest", + "moby": true + } + }, + "extensions": [ + "ms-python.black-formatter", + "donjayamanne.python-extension-pack", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance", + "eamodio.gitlens", + "wholroyd.jinja", + "pmbenjamin.vscode-snyk", + "visualstudioexptteam.vscodeintellicode", + "yzhang.markdown-all-in-one", + "ms-ossdata.vscode-postgresql", + "GitHub.copilot", + "ms-vsliveshare.vsliveshare", + "mtxr.sqltools", + "mtxr.sqltools-driver-pg", + ], + "postCreateCommand": "notify-worker-entrypoint.sh", + "remoteUser": "vscode" +} \ No newline at end of file diff --git a/docker-compose.devcontainer.yml b/docker-compose.devcontainer.yml index fd8da349e..62bdfad51 100644 --- a/docker-compose.devcontainer.yml +++ b/docker-compose.devcontainer.yml @@ -24,7 +24,7 @@ services: container_name: redis image: redis:6.2 restart: always - command: redis-server --port 6380 + command: sh -c "rm -f /data/dump.rdb && redis-server --port 6380" # disable persistence ports: - "6380:6380" expose: diff --git a/migrations/versions/0029_fix_email_from.py b/migrations/versions/0029_fix_email_from.py index 52d048c69..a2bb3f50c 100644 --- a/migrations/versions/0029_fix_email_from.py +++ b/migrations/versions/0029_fix_email_from.py @@ -16,8 +16,8 @@ import sqlalchemy as sa service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' def upgrade(): op.get_bind() - op.execute("update services set email_from = 'gov.uk.notify' where id = '{}'".format(service_id)) - op.execute("update services_history set email_from = 'gov.uk.notify' where id = '{}'".format(service_id)) + op.execute("update services set email_from = 'testsender' where id = '{}'".format(service_id)) + op.execute("update services_history set email_from = 'testsender' where id = '{}'".format(service_id)) def downgrade(): diff --git a/sample.env b/sample.env index 3edac2c06..eba3714f2 100644 --- a/sample.env +++ b/sample.env @@ -20,10 +20,10 @@ SECRET_KEY=dev-notify-secret-key DANGEROUS_SALT=dev-notify-salt # URL of admin app -ADMIN_BASE_URL=http://admin:6012 +ADMIN_BASE_URL=http://0.0.0.0:6012 # URL of api app (on AWS this is the internal api endpoint) -API_HOST_NAME=http://dev:6011 +API_HOST_NAME=http://0.0.0.0:6011 # URL of redis instance REDIS_URL=redis://redis:6380