diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index df553a7da..937cbd887 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,27 +7,15 @@ permissions: env: DEBUG: True - ANTIVIRUS_ENABLED: 0 NOTIFY_ENVIRONMENT: test - STATSD_HOST: localhost - SES_STUB_URL: None - NOTIFY_APP_NAME: api - NOTIFY_EMAIL_DOMAIN: dispostable.com - NOTIFY_LOG_PATH: /workspace/logs/app.log - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - REDIS_URL: redis://localhost:6380 - REDIS_ENABLED: False + NOTIFY_EMAIL_DOMAIN: dispostable.com + REDIS_ENABLED: 0 AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 - AWS_ACCESS_KEY_ID: not-a-real-key-id - AWS_SECRET_ACCESS_KEY: not-a-real-secret jobs: build: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index f10e435bc..99e47b7db 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -11,22 +11,12 @@ permissions: env: DEBUG: True - ANTIVIRUS_ENABLED: 0 NOTIFY_ENVIRONMENT: test - STATSD_HOST: localhost - SES_STUB_URL: None - NOTIFY_APP_NAME: api - NOTIFY_EMAIL_DOMAIN: dispostable.com - NOTIFY_LOG_PATH: /workspace/logs/app.log - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - REDIS_URL: redis://localhost:6380 - REDIS_ENABLED: False + NOTIFY_EMAIL_DOMAIN: dispostable.com + REDIS_ENABLED: 0 AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 diff --git a/.gitignore b/.gitignore index a15cb5989..ede52acef 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ test_results.xml # Django stuff: *.log +/logs/* # Sphinx documentation docs/_build/ diff --git a/Makefile b/Makefile index b14f342fa..9ced0aaa5 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ bootstrap: ## Set up everything to run the app bootstrap-with-docker: ## Build the image to run the app in Docker docker build -f docker/Dockerfile -t notifications-api . +.PHONY: run-procfile +run-procfile: + pipenv run honcho start -f Procfile.dev + .PHONY: run-flask run-flask: ## Run flask pipenv run flask run -p 6011 --host=0.0.0.0 diff --git a/Pipfile b/Pipfile index 3af486903..0ae471394 100644 --- a/Pipfile +++ b/Pipfile @@ -77,6 +77,7 @@ requests-mock = "==1.9.3" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} pip-audit = "*" bandit = "*" +honcho = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 883e22816..53ac2cf4d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f4f8c5db036bc6412ed4651902b4d8d5906fc920063bf0182933f3947959d120" + "sha256": "b1e61b613cb208ab42359fe335224367647de3b568adf4ed0872976d738a054b" }, "pipfile-spec": 6, "requires": { @@ -963,10 +963,10 @@ }, "pytz": { "hashes": [ - "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", - "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427", + "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2" ], - "version": "==2022.5" + "version": "==2022.6" }, "pyyaml": { "hashes": [ @@ -1581,6 +1581,14 @@ "markers": "python_version >= '3.7'", "version": "==3.1.29" }, + "honcho": { + "hashes": [ + "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f", + "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9" + ], + "index": "pypi", + "version": "==1.1.0" + }, "html5lib": { "hashes": [ "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", @@ -1931,10 +1939,10 @@ }, "pytz": { "hashes": [ - "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", - "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427", + "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2" ], - "version": "==2022.5" + "version": "==2022.6" }, "pyyaml": { "hashes": [ diff --git a/Procfile.dev b/Procfile.dev new file mode 100644 index 000000000..bc4af912c --- /dev/null +++ b/Procfile.dev @@ -0,0 +1,2 @@ +web: make run-flask +worker: make run-celery diff --git a/README.md b/README.md index ed1e149d5..b8cd421c0 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,19 @@ Our other repositories are: # follow the instructions in .env ``` -1. Run Flask +1. Run the web server and background worker - `make run-flask` + `make run-procfile` -1. Run Celery +1. Or run them individually: - `make run-celery` + * Run Flask (web server) + + `make run-flask` + + * Run Celery (background worker) + + `make run-celery` ### VS Code && Docker installation @@ -104,4 +110,4 @@ Work through [commit `e604385`](https://github.com/GSA/notifications-api/commit/ ## Contributing -As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. \ No newline at end of file +As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. diff --git a/app/aws/s3.py b/app/aws/s3.py index 7312f4cbe..7a705c544 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -41,21 +41,21 @@ def file_exists( def get_job_location(service_id, job_id): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + current_app.config['CSV_UPLOAD_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, job_id), - current_app.config['CSV_UPLOAD_ACCESS_KEY'], - current_app.config['CSV_UPLOAD_SECRET_KEY'], - current_app.config['CSV_UPLOAD_REGION'], + current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'], + current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'], + current_app.config['CSV_UPLOAD_BUCKET']['region'], ) def get_contact_list_location(service_id, contact_list_id): return ( - current_app.config['CONTACT_LIST_BUCKET_NAME'], + current_app.config['CONTACT_LIST_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id), - current_app.config['CONTACT_LIST_ACCESS_KEY'], - current_app.config['CONTACT_LIST_SECRET_KEY'], - current_app.config['CONTACT_LIST_REGION'], + current_app.config['CONTACT_LIST_BUCKET']['access_key_id'], + current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'], + current_app.config['CONTACT_LIST_BUCKET']['region'], ) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 533f3d6af..1218e8852 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,41 +2,30 @@ import json import os -def find_by_service_name(services, service_name): - for i in range(len(services)): - if services[i]['name'] == service_name: - return services[i] - return None +class CloudfoundryConfig: + def __init__(self): + self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}') + buckets = self.parsed_services.get('s3') or [] + self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets} + self._empty_bucket_credentials = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + @property + def redis_url(self): + try: + return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace( + 'redis://', + 'rediss://' + ) + except KeyError: + return os.environ.get('REDIS_URL') + + def s3_credentials(self, service_name): + return self.s3_buckets.get(service_name) or self._empty_bucket_credentials -def extract_cloudfoundry_config(): - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - - # Postgres config - os.environ['SQLALCHEMY_DATABASE_URI'] = \ - vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql') - # Redis config - os.environ['REDIS_URL'] = \ - vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://') - - # CSV Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] - - # Contact List Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region'] +cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index d70ec913c..723143c4f 100644 --- a/app/config.py +++ b/app/config.py @@ -5,12 +5,7 @@ from datetime import timedelta from celery.schedules import crontab from kombu import Exchange, Queue -if os.environ.get('VCAP_SERVICES'): - # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate - # standard environment variables from it - from app.cloudfoundry_config import extract_cloudfoundry_config - - extract_cloudfoundry_config() +from app.cloudfoundry_config import cloud_config class QueueNames(object): @@ -68,76 +63,31 @@ class TaskNames(object): class Config(object): + NOTIFY_APP_NAME = 'api' + NOTIFY_ENVIRONMENT = os.environ.get('NOTIFY_ENVIRONMENT', 'development') # URL of admin app - ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL') - + ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL', 'http://localhost:6012') # URL of api app (on AWS this is the internal api endpoint) - API_HOST_NAME = os.environ.get('API_HOST_NAME') + API_HOST_NAME = os.environ.get('API_HOST_NAME', 'http://localhost:6011') + # Credentials # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API - ADMIN_CLIENT_ID = 'notify-admin' - + # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match + ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin') INTERNAL_CLIENT_API_KEYS = json.loads( - os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') - ) # TODO: handled by varsfile? - + os.environ.get( + 'INTERNAL_CLIENT_API_KEYS', + ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, os.getenv('ADMIN_CLIENT_SECRET'))) + ) + ) # encyption secret/salt - ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') SECRET_KEY = os.environ.get('SECRET_KEY') DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') + ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') + ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') - # DB conection string + # DB settings SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') - - # AWS SMS - AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") - AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") - - # MMG API Key - MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') - - # Firetext API Key - FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") - FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") - - # Whether to ignore POSTs from SNS for replies to SMS we sent - RECEIVE_INBOUND_SMS = False - - # Use notify.sandbox.10x sending domain unless overwritten by environment - NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' - - # AWS SNS topics for delivery receipts - VALIDATE_SNS_TOPICS = True - VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] - - # URL of redis instance - REDIS_URL = os.environ.get('REDIS_URL') - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - EXPIRE_CACHE_TEN_MINUTES = 600 - EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 - - # Zendesk - ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') - - # Logging - DEBUG = False - NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH') - - # Cronitor - CRONITOR_ENABLED = False - CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}')) - - # Antivirus - ANTIVIRUS_ENABLED = True - - ########################### - # Default config values ### - ########################### - - NOTIFY_ENVIRONMENT = 'development' - AWS_REGION = 'us-west-2' - INVITATION_EXPIRATION_DAYS = 2 - NOTIFY_APP_NAME = 'api' SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5)) @@ -146,24 +96,68 @@ class Config(object): SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 - TEST_MESSAGE_FILENAME = 'Test message' - ONE_OFF_MESSAGE_FILENAME = 'Report' - MAX_VERIFY_CODE_COUNT = 5 - MAX_FAILED_LOGIN_COUNT = 10 + REDIS_URL = cloud_config.redis_url + REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1' + EXPIRE_CACHE_TEN_MINUTES = 600 + EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 + # AWS Settings + AWS_REGION = os.environ.get('AWS_REGION') + AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") + AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") + # Whether to ignore POSTs from SNS for replies to SMS we sent + RECEIVE_INBOUND_SMS = False + NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service + # AWS SNS topics for delivery receipts + VALIDATE_SNS_TOPICS = True + VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] - # be careful increasing this size without being sure that we won't see slowness in pysftp - MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb - MAX_LETTER_PDF_COUNT_PER_ZIP = 500 - - CHECK_PROXY_HEADER = False - + # SMS config to be cleaned up during https://github.com/GSA/notifications-api/issues/7 + # MMG API Key + MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') + # Firetext API Key + FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") + FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # these should always add up to 100% SMS_PROVIDER_RESTING_POINTS = { 'mmg': 50, 'firetext': 50 } + FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) + MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) + MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) + MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") + FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") + + # Zendesk + ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') + + # Logging + DEBUG = False + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'logs/application.log') + + # Monitoring + CRONITOR_ENABLED = False + CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}')) + STATSD_HOST = os.environ.get('STATSD_HOST') + STATSD_PORT = 8125 + STATSD_ENABLED = bool(STATSD_HOST) + + # Antivirus + ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED', '1') == '1' + + SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days + INVITATION_EXPIRATION_DAYS = 2 + TEST_MESSAGE_FILENAME = 'Test message' + ONE_OFF_MESSAGE_FILENAME = 'Report' + MAX_VERIFY_CODE_COUNT = 5 + MAX_FAILED_LOGIN_COUNT = 10 + API_RATE_LIMIT_ENABLED = True + + # be careful increasing this size without being sure that we won't see slowness in pysftp + MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb + MAX_LETTER_PDF_COUNT_PER_ZIP = 500 NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' @@ -334,29 +328,15 @@ class Config(object): FROM_NUMBER = 'development' - STATSD_HOST = os.environ.get('STATSD_HOST') - STATSD_PORT = 8125 - STATSD_ENABLED = bool(STATSD_HOST) - - SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days - SIMULATED_EMAIL_ADDRESSES = ( 'simulate-delivered@notifications.service.gov.uk', 'simulate-delivered-2@notifications.service.gov.uk', 'simulate-delivered-3@notifications.service.gov.uk', ) - SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') FREE_SMS_TIER_FRAGMENT_COUNT = 250000 - SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) - FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) - ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') - ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') - HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') @@ -365,74 +345,40 @@ class Config(object): DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') - # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` - MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") - FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") - AWS_REGION = 'us-west-2' +def _default_s3_credentials(bucket_name): + return { + 'bucket': bucket_name, + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } -###################### -# Config overrides ### -###################### - class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False - - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2') - CONTACT_LIST_BUCKET_NAME = 'local-contact-list' - CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2') - # TEST_LETTERS_BUCKET_NAME = 'development-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp' - # LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise' - - # INTERNAL_CLIENT_API_KEYS = { - # Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - # } - - SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development - DANGEROUS_SALT = 'dev-notify-salt' - - MMG_INBOUND_SMS_AUTH = ['testkey'] - MMG_INBOUND_SMS_USERNAME = ['username'] - - NOTIFY_ENVIRONMENT = 'development' - NOTIFY_LOG_PATH = 'application.log' - - NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') - - SQLALCHEMY_DATABASE_URI = os.environ.get( - 'SQLALCHEMY_DATABASE_URI', - 'postgresql://postgres:chummy@db:5432/notification_api' - ) - - ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' - - ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012') - - API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011') - - API_RATE_LIMIT_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] + # Buckets + CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') + + # credential overrides + DANGEROUS_SALT = 'dev-notify-salt' + SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development + INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']} + class Test(Development): - NOTIFY_EMAIL_DOMAIN = 'test.notify.com' FROM_NUMBER = 'testing' - NOTIFY_ENVIRONMENT = 'test' TESTING = True + ANTIVIRUS_ENABLED = True + DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] + + FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] + MMG_INBOUND_SMS_AUTH = ['testkey'] + MMG_INBOUND_SMS_USERNAME = ['username'] HIGH_VOLUME_SERVICE = [ '941b6f9a-50d7-4742-8d50-f365ca74bf27', @@ -441,139 +387,38 @@ class Test(Development): '10d1b9c9-0072-4fa9-ae1c-595e333841da', ] - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'test-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'test-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp' - # LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' + CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') # this is overriden in CI - SQLALCHEMY_DATABASE_URI = os.getenv( - 'SQLALCHEMY_DATABASE_TEST_URI', - 'postgresql://postgres:chummy@db:5432/test_notification_api' - ) + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI') CELERY = { **Config.CELERY, 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' } - ANTIVIRUS_ENABLED = True - - API_RATE_LIMIT_ENABLED = True - API_HOST_NAME = "http://localhost:6011" - - SMS_INBOUND_WHITELIST = ['203.0.113.195'] - FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999' - MMG_URL = 'https://example.com/mmg' - FIRETEXT_URL = 'https://example.com/firetext' - DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] - - -class Preview(Config): - NOTIFY_EMAIL_DOMAIN = 'notify.works' - NOTIFY_ENVIRONMENT = 'preview' - CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'preview-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'preview-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise' - FROM_NUMBER = 'preview' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = False - - -class Staging(Config): - NOTIFY_EMAIL_DOMAIN = 'staging-notify.works' - NOTIFY_ENVIRONMENT = 'staging' - CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'staging-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'staging-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise' - FROM_NUMBER = 'stage' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = True - - -class Live(Config): - NOTIFY_ENVIRONMENT = 'live' +class Production(Config): # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get( - 'CSV_UPLOAD_BUCKET_NAME', - 'notifications-prototype-csv-upload' - ) # created in gsa sandbox - CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') - CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') - CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_BUCKET_NAME = os.environ.get( - 'CONTACT_LIST_BUCKET_NAME', - 'notifications-prototype-contact-list-upload' - ) # created in gsa sandbox - CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') - CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') - CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION') - # TODO: verify below buckets only used for letters - # TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox - # DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox - # LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox - # LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox - # INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox - # TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox - # LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox + CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}") + CONTACT_LIST_BUCKET = cloud_config.s3_credentials( + f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}") FROM_NUMBER = 'US Notify' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = True - SES_STUB_URL = None CRONITOR_ENABLED = True - # DEBUG = True - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log') - - -class CloudFoundryConfig(Config): +class Staging(Production): pass -# CloudFoundry sandbox -class Sandbox(CloudFoundryConfig): - NOTIFY_EMAIL_DOMAIN = 'notify.works' - NOTIFY_ENVIRONMENT = 'sandbox' - CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list' - # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' - # TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf' - FROM_NUMBER = 'sandbox' - - configs = { 'development': Development, 'test': Test, - 'live': Live, - 'production': Live, 'staging': Staging, - 'preview': Preview, - 'sandbox': Sandbox + 'production': Production } diff --git a/logs/.keep b/logs/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/logs/app.log.json b/logs/app.log.json deleted file mode 100644 index 2ac3ecd7b..000000000 --- a/logs/app.log.json +++ /dev/null @@ -1,25 +0,0 @@ -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:41:53", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:42:30", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:02", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:05", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:59:55", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:38", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:52", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:56", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Email already registered for user 7796f6d5-4fba-48d5-bfa1-80c5b3465d24", "pathname": "/workspace/app/user/rest.py", "lineno": 415, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "To email is testreceiver@dispostable.com", "pathname": "/workspace/app/user/rest.py", "lineno": 418, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "template.id is 0880fbb1-a0c6-46f0-9a8e-36c986381ceb", "pathname": "/workspace/app/user/rest.py", "lineno": 423, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "service.id is d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/user/rest.py", "lineno": 424, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 108, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with id 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 114, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with to address: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 139, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with type: email", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 149, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification to formatted email: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 151, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Firing dao_create_notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 159, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis enabled, querying cache key for service id: d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 162, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key: d6aa2c68-a2d9-4437-ab19-3ae8eb202553-2022-06-29-count", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 164, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key does not exist", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 166, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Set redis daily limit cache key to 1", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 172, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "email 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4 created at 2022-06-29 06:07:57.373123", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 177, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Sending notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 442, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Sent notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 446, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} diff --git a/manifest.yml b/manifest.yml index c07573fc6..77ee86a1c 100644 --- a/manifest.yml +++ b/manifest.yml @@ -32,16 +32,13 @@ applications: NOTIFY_LOG_PATH: /home/vcap/logs/app.log FLASK_APP: application.py FLASK_ENV: production - DEPLOY_ENV: ((env)) - NOTIFY_ENVIRONMENT: live + NOTIFY_ENVIRONMENT: ((env)) API_HOST_NAME: https://notifications-api.app.cloud.gov ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov - STATSD_HOST: localhost # Credentials variables INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' - ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) @@ -49,5 +46,3 @@ applications: AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 - - DVLA_EMAIL_ADDRESSES: [] diff --git a/sample.env b/sample.env index 3909b9beb..ca51a6007 100644 --- a/sample.env +++ b/sample.env @@ -1,12 +1,12 @@ # STEPS TO SET UP -# +# # 1. Pull down AWS creds from cloud.gov using `cf env`, then update AWS section -# +# # 2. Uncomment either the Docker setup or the direct setup # # 3. Comment out the other setup -# -# 4. Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) +# +# 4. If needed, set `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) # # 5. Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values # @@ -16,9 +16,9 @@ ############################################################# # AWS -AWS_REGION=us-west-2 AWS_ACCESS_KEY_ID="don't write secrets to the sample file" AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" +AWS_REGION=us-west-2 AWS_PINPOINT_REGION=us-west-2 AWS_US_TOLL_FREE_NUMBER=+18446120782 @@ -28,7 +28,6 @@ AWS_US_TOLL_FREE_NUMBER=+18446120782 ADMIN_BASE_URL=http://admin:6012 API_HOST_NAME=http://dev:6011 REDIS_URL=redis://redis:6380 -REDIS_ENABLED=1 SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api @@ -36,7 +35,6 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat # ADMIN_BASE_URL=http://localhost:6012 # API_HOST_NAME=http://localhost:6011 # REDIS_URL=redis://localhost:6379 -# REDIS_ENABLED=1 # SQLALCHEMY_DATABASE_URI=postgresql://localhost:5432/notification_api # SQLALCHEMY_DATABASE_TEST_URI=postgresql://localhost:5432/test_notification_api @@ -45,12 +43,12 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat # Debug DEBUG=True ANTIVIRUS_ENABLED=0 +REDIS_ENABLED=1 NOTIFY_ENVIRONMENT=development STATSD_HOST=localhost SES_STUB_URL=None NOTIFY_APP_NAME=api -NOTIFY_EMAIL_DOMAIN=dispostable.com -NOTIFY_LOG_PATH=/workspace/logs/app.log +# NOTIFY_EMAIL_DOMAIN=notify.sandbox.10x.gsa.gov ############################################################# @@ -58,9 +56,3 @@ NOTIFY_LOG_PATH=/workspace/logs/app.log FLASK_APP=application.py FLASK_ENV=development WERKZEUG_DEBUG_PIN=off -SECRET_KEY=dev-notify-secret-key -DANGEROUS_SALT=dev-notify-salt - -# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API -ADMIN_CLIENT_ID=notify-admin -ADMIN_CLIENT_SECRET=dev-notify-secret-key diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py index e0931d31f..4cc6c4baa 100644 --- a/tests/app/celery/test_ftp_update_tasks.py +++ b/tests/app/celery/test_ftp_update_tasks.py @@ -92,9 +92,9 @@ def test_update_letter_notifications_statuses_calls_with_correct_bucket_location s3_mock.assert_called_with('{}-ftp'.format( current_app.config['NOTIFY_EMAIL_DOMAIN']), 'NOTIFY-20170823160812-RSP.TXT', - os.environ['AWS_ACCESS_KEY_ID'], - os.environ['AWS_SECRET_ACCESS_KEY'], - os.environ['AWS_REGION'], + os.environ.get('AWS_ACCESS_KEY_ID'), + os.environ.get('AWS_SECRET_ACCESS_KEY'), + os.environ.get('AWS_REGION'), ) diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 1d5db39d7..80fad2648 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -3,7 +3,14 @@ import os import pytest -from app.cloudfoundry_config import extract_cloudfoundry_config +from app.cloudfoundry_config import CloudfoundryConfig + +bucket_credentials = { + 'access_key_id': 'csv-access', + 'bucket': 'csv-upload-bucket', + 'region': 'us-gov-west-1', + 'secret_access_key': 'csv-secret' +} @pytest.fixture @@ -22,12 +29,7 @@ def vcap_services(): 's3': [ { 'name': 'notifications-api-csv-upload-bucket-test', - 'credentials': { - 'access_key_id': 'csv-access', - 'bucket': 'csv-upload-bucket', - 'region': 'us-gov-west-1', - 'secret_access_key': 'csv-secret' - } + 'credentials': bucket_credentials }, { 'name': 'notifications-api-contact-list-bucket-test', @@ -43,12 +45,33 @@ def vcap_services(): } -def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): - os.environ['DEPLOY_ENV'] = 'test' +def test_redis_url(vcap_services): os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) - extract_cloudfoundry_config() - assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri' - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' - assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' + assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379' + + +def test_redis_url_falls_back_to_REDIS_URL(): + expected = 'redis://yyy:6379' + os.environ['REDIS_URL'] = expected + os.environ['VCAP_SERVICES'] = "" + + assert CloudfoundryConfig().redis_url == expected + + +def test_s3_bucket_credentials(vcap_services): + os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) + + assert CloudfoundryConfig().s3_credentials('notifications-api-csv-upload-bucket-test') == bucket_credentials + + +def test_s3_bucket_credentials_falls_back_to_empty_creds(): + os.environ['VCAP_SERVICES'] = "" + expected = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + assert CloudfoundryConfig().s3_credentials('bucket') == expected diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 17bb96bb6..1f77278d1 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -1,62 +1,6 @@ -import importlib -import os -from unittest import mock - -import pytest - -from app import config from app.config import QueueNames -def cf_conf(): - os.environ['ADMIN_BASE_URL'] = 'cf' - - -@pytest.fixture -def reload_config(): - """ - Reset config, by simply re-running config.py from a fresh environment - """ - old_env = os.environ.copy() - - yield - - os.environ.clear() - for k, v in old_env.items(): - os.environ[k] = v - - importlib.reload(config) - - -def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ['VCAP_SERVICES'] = 'some json blob' - os.environ['VCAP_APPLICATION'] = 'some json blob' - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'cf' - assert config.Config.ADMIN_BASE_URL == 'cf' - - -def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ.pop('VCAP_SERVICES', None) - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert not cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'env' - assert config.Config.ADMIN_BASE_URL == 'env' - - def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues()