From d55c099043db6b9e3fd6af21384121d00b606be2 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 28 Oct 2022 16:20:41 -0400 Subject: [PATCH 01/12] update licensing situation for us gov --- LICENSE | 22 ---------------------- README.md | 7 +++++++ 2 files changed, 7 insertions(+), 22 deletions(-) delete mode 100644 LICENSE diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 50c5287af..000000000 --- a/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Government Digital Service - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/README.md b/README.md index 79a682726..ed1e149d5 100644 --- a/README.md +++ b/README.md @@ -98,3 +98,10 @@ If you're working in VS Code, you can also leverage Docker for a containerized d NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container. +## License && public domain + +Work through [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) is licensed by the UK government until the MIT license. Work after that commit is in the worldwide public domain. See [LICENSE.md](./LICENSE.md) for more information. + +## Contributing + +As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. \ No newline at end of file From 92f59ed600f6dcc84e78315211719d10959d08b4 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 28 Oct 2022 16:22:59 -0400 Subject: [PATCH 02/12] include new/renamed files --- CONTRIBUTING.md | 19 ++++++++++++++++++ LICENSE.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..c2792b81f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,19 @@ +# Welcome! + +We're so glad you're thinking about contributing to a [open source project of the U.S. government](https://code.gov/)! If you're unsure about anything, just ask -- or submit the issue or pull request anyway. The worst that can happen is you'll be politely asked to change something. We love all friendly contributions. + +We encourage you to read this project's CONTRIBUTING policy (you are here), its [LICENSE](LICENSE.md), and its [README](README.md). + +## Policies + +We want to ensure a welcoming environment for all of our projects. Our staff follow the [TTS Code of Conduct](https://18f.gsa.gov/code-of-conduct/) and all contributors should do the same. + +This project is adhering to the [18F Open Source Policy](https://github.com/18f/open-source-policy). + +As part of a U.S. government agency, the General Services Administration (GSA)’s Technology Transformation Services (TTS) takes seriously our responsibility to protect the public’s information, including financial and personal information, from unwarranted disclosure. For more information about security and vulnerability disclosure for our projects, please read our [18F Vulnerability Disclosure Policy](https://18f.gsa.gov/vulnerability-disclosure-policy/). + +## Public domain + +This project is in the public domain within the United States, and applicable copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest. \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 000000000..97bd78bc0 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,53 @@ +# License + +## A few parts of this project are not in the public domain + +### Copyright ownership of Works created by U.S. federal employees + +This repository contains both the original work for [GOV.UK Notify](https://github.com/alphagov/notifications-api) (the Software, as defined in the MIT License) and modifications made by the General Services Administration (GSA). This repository combines the Software and these GSA modifications, which is a “joint work” of the Software and modifications combined into a single work. + +As a work of the United States Government, the software modifications made by GSA are not subject to copyright within the United States. Additionally, GSA waives copyright and related rights in its software modifications worldwide through the [CC0 1.0 Universal Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +The Software remains subject to copyright under the MIT License. This consists of all work until [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) on June 1, 2022. + +#### Full license text for the MIT licensed files: + +``` +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` + +## The rest of this project is in the worldwide public domain + +As a work of the United States government, this project is in the public domain within the United States. + +Additionally, we waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +### CC0 1.0 Universal Summary + +This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). + +#### No copyright + +The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. + +You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission. + +#### Other information + +In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights. + +Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer. + +## Contributions to this project + +As stated in [CONTRIBUTING](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. + +## Trademark + +GSA owns, reserves, and retains all rights, title, and interest in all trademarks owned by GSA. The User does not acquire any right or interest of any kind in any GSA trademark because of their use of the Modified Version or any of the modifications made to the Work by GSA. \ No newline at end of file From e4be1850ef2b89c8fe442c91767c87b60b9d550e Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 31 Oct 2022 10:34:37 -0400 Subject: [PATCH 03/12] Remove stray log file --- .gitignore | 1 + logs/.keep | 0 logs/app.log.json | 25 ------------------------- 3 files changed, 1 insertion(+), 25 deletions(-) create mode 100644 logs/.keep delete mode 100644 logs/app.log.json diff --git a/.gitignore b/.gitignore index a15cb5989..ede52acef 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ test_results.xml # Django stuff: *.log +/logs/* # Sphinx documentation docs/_build/ diff --git a/logs/.keep b/logs/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/logs/app.log.json b/logs/app.log.json deleted file mode 100644 index 2ac3ecd7b..000000000 --- a/logs/app.log.json +++ /dev/null @@ -1,25 +0,0 @@ -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:41:53", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:42:30", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:02", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:05", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:59:55", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:38", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:52", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:56", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Email already registered for user 7796f6d5-4fba-48d5-bfa1-80c5b3465d24", "pathname": "/workspace/app/user/rest.py", "lineno": 415, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "To email is testreceiver@dispostable.com", "pathname": "/workspace/app/user/rest.py", "lineno": 418, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "template.id is 0880fbb1-a0c6-46f0-9a8e-36c986381ceb", "pathname": "/workspace/app/user/rest.py", "lineno": 423, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "service.id is d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/user/rest.py", "lineno": 424, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 108, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with id 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 114, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with to address: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 139, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification with type: email", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 149, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Presisting notification to formatted email: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 151, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Firing dao_create_notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 159, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis enabled, querying cache key for service id: d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 162, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key: d6aa2c68-a2d9-4437-ab19-3ae8eb202553-2022-06-29-count", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 164, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key does not exist", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 166, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Set redis daily limit cache key to 1", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 172, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "email 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4 created at 2022-06-29 06:07:57.373123", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 177, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Sending notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 442, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} -{"name": "app", "levelname": "INFO", "message": "Sent notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 446, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} From 169a637aa33bc86d53e9b80cd9da1709e7a5f2ac Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 31 Oct 2022 12:18:25 -0400 Subject: [PATCH 04/12] use setup action in deploy workflow --- .github/workflows/deploy.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b3946fb48..588b99c4f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,12 +16,6 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - - name: Install container dependencies - run: | - sudo apt-get update \ - && sudo apt-get install -y --no-install-recommends \ - libcurl4-openssl-dev - - uses: actions/checkout@v3 with: fetch-depth: 2 @@ -48,11 +42,7 @@ jobs: TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} run: terraform apply -auto-approve -input=false - - name: Set up Python 3.9 - uses: actions/setup-python@v3 - with: - python-version: "3.9" - + - uses: ./.github/actions/setup-project - name: Install application dependencies run: make bootstrap From 7aafdd7bac6283e15390817d5b8cecddb7874f8b Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 31 Oct 2022 13:25:59 -0400 Subject: [PATCH 05/12] Clean up config settings --- .github/workflows/checks.yml | 16 +- .github/workflows/daily_checks.yml | 14 +- app/config.py | 322 ++++++++--------------------- sample.env | 18 +- 4 files changed, 96 insertions(+), 274 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index df553a7da..937cbd887 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,27 +7,15 @@ permissions: env: DEBUG: True - ANTIVIRUS_ENABLED: 0 NOTIFY_ENVIRONMENT: test - STATSD_HOST: localhost - SES_STUB_URL: None - NOTIFY_APP_NAME: api - NOTIFY_EMAIL_DOMAIN: dispostable.com - NOTIFY_LOG_PATH: /workspace/logs/app.log - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - REDIS_URL: redis://localhost:6380 - REDIS_ENABLED: False + NOTIFY_EMAIL_DOMAIN: dispostable.com + REDIS_ENABLED: 0 AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 - AWS_ACCESS_KEY_ID: not-a-real-key-id - AWS_SECRET_ACCESS_KEY: not-a-real-secret jobs: build: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index f10e435bc..99e47b7db 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -11,22 +11,12 @@ permissions: env: DEBUG: True - ANTIVIRUS_ENABLED: 0 NOTIFY_ENVIRONMENT: test - STATSD_HOST: localhost - SES_STUB_URL: None - NOTIFY_APP_NAME: api - NOTIFY_EMAIL_DOMAIN: dispostable.com - NOTIFY_LOG_PATH: /workspace/logs/app.log - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - REDIS_URL: redis://localhost:6380 - REDIS_ENABLED: False + NOTIFY_EMAIL_DOMAIN: dispostable.com + REDIS_ENABLED: 0 AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 diff --git a/app/config.py b/app/config.py index d70ec913c..86f0be77b 100644 --- a/app/config.py +++ b/app/config.py @@ -68,76 +68,30 @@ class TaskNames(object): class Config(object): + NOTIFY_APP_NAME = 'api' + NOTIFY_ENVIRONMENT = os.environ.get('NOTIFY_ENVIRONMENT', 'development') # URL of admin app - ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL') - + ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL', 'http://localhost:6012') # URL of api app (on AWS this is the internal api endpoint) - API_HOST_NAME = os.environ.get('API_HOST_NAME') + API_HOST_NAME = os.environ.get('API_HOST_NAME', 'http://localhost:6011') + # Credentials # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API - ADMIN_CLIENT_ID = 'notify-admin' - + ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID') INTERNAL_CLIENT_API_KEYS = json.loads( - os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') - ) # TODO: handled by varsfile? - + os.environ.get( + 'INTERNAL_CLIENT_API_KEYS', + ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, os.getenv('ADMIN_CLIENT_SECRET'))) + ) + ) # encyption secret/salt - ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') SECRET_KEY = os.environ.get('SECRET_KEY') DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') + ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') + ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') - # DB conection string + # DB settings SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') - - # AWS SMS - AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") - AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") - - # MMG API Key - MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') - - # Firetext API Key - FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") - FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") - - # Whether to ignore POSTs from SNS for replies to SMS we sent - RECEIVE_INBOUND_SMS = False - - # Use notify.sandbox.10x sending domain unless overwritten by environment - NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' - - # AWS SNS topics for delivery receipts - VALIDATE_SNS_TOPICS = True - VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] - - # URL of redis instance - REDIS_URL = os.environ.get('REDIS_URL') - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - EXPIRE_CACHE_TEN_MINUTES = 600 - EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 - - # Zendesk - ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') - - # Logging - DEBUG = False - NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH') - - # Cronitor - CRONITOR_ENABLED = False - CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}')) - - # Antivirus - ANTIVIRUS_ENABLED = True - - ########################### - # Default config values ### - ########################### - - NOTIFY_ENVIRONMENT = 'development' - AWS_REGION = 'us-west-2' - INVITATION_EXPIRATION_DAYS = 2 - NOTIFY_APP_NAME = 'api' SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5)) @@ -146,24 +100,68 @@ class Config(object): SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 - TEST_MESSAGE_FILENAME = 'Test message' - ONE_OFF_MESSAGE_FILENAME = 'Report' - MAX_VERIFY_CODE_COUNT = 5 - MAX_FAILED_LOGIN_COUNT = 10 + REDIS_URL = os.environ.get('REDIS_URL') + REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1' + EXPIRE_CACHE_TEN_MINUTES = 600 + EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 + # AWS Settings + AWS_REGION = os.environ.get('AWS_REGION') + AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION") + AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER") + # Whether to ignore POSTs from SNS for replies to SMS we sent + RECEIVE_INBOUND_SMS = False + NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service + # AWS SNS topics for delivery receipts + VALIDATE_SNS_TOPICS = True + VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] - # be careful increasing this size without being sure that we won't see slowness in pysftp - MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb - MAX_LETTER_PDF_COUNT_PER_ZIP = 500 - - CHECK_PROXY_HEADER = False - + # SMS config to be cleaned up during https://github.com/GSA/notifications-api/issues/7 + # MMG API Key + MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder') + # Firetext API Key + FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") + FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") # these should always add up to 100% SMS_PROVIDER_RESTING_POINTS = { 'mmg': 50, 'firetext': 50 } + FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) + MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) + MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) + MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") + FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") + + # Zendesk + ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') + + # Logging + DEBUG = False + NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'logs/application.log') + + # Monitoring + CRONITOR_ENABLED = False + CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}')) + STATSD_HOST = os.environ.get('STATSD_HOST') + STATSD_PORT = 8125 + STATSD_ENABLED = bool(STATSD_HOST) + + # Antivirus + ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED', '1') == '1' + + SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days + INVITATION_EXPIRATION_DAYS = 2 + TEST_MESSAGE_FILENAME = 'Test message' + ONE_OFF_MESSAGE_FILENAME = 'Report' + MAX_VERIFY_CODE_COUNT = 5 + MAX_FAILED_LOGIN_COUNT = 10 + API_RATE_LIMIT_ENABLED = True + + # be careful increasing this size without being sure that we won't see slowness in pysftp + MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb + MAX_LETTER_PDF_COUNT_PER_ZIP = 500 NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' @@ -334,29 +332,15 @@ class Config(object): FROM_NUMBER = 'development' - STATSD_HOST = os.environ.get('STATSD_HOST') - STATSD_PORT = 8125 - STATSD_ENABLED = bool(STATSD_HOST) - - SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days - SIMULATED_EMAIL_ADDRESSES = ( 'simulate-delivered@notifications.service.gov.uk', 'simulate-delivered-2@notifications.service.gov.uk', 'simulate-delivered-3@notifications.service.gov.uk', ) - SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') FREE_SMS_TIER_FRAGMENT_COUNT = 250000 - SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]')) - FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]')) - MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]')) - ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') - ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') - HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]')) TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') @@ -365,23 +349,13 @@ class Config(object): DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000') DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') - # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env` - MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php") - FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json") - - AWS_REGION = 'us-west-2' - - -###################### -# Config overrides ### -###################### class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False + DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - + # Buckets CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') @@ -390,49 +364,24 @@ class Development(Config): CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2') - # TEST_LETTERS_BUCKET_NAME = 'development-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp' - # LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise' - # INTERNAL_CLIENT_API_KEYS = { - # Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - # } - - SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development + # credential overrides DANGEROUS_SALT = 'dev-notify-salt' - - MMG_INBOUND_SMS_AUTH = ['testkey'] - MMG_INBOUND_SMS_USERNAME = ['username'] - - NOTIFY_ENVIRONMENT = 'development' - NOTIFY_LOG_PATH = 'application.log' - - NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') - - SQLALCHEMY_DATABASE_URI = os.environ.get( - 'SQLALCHEMY_DATABASE_URI', - 'postgresql://postgres:chummy@db:5432/notification_api' - ) - - ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' - - ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012') - - API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011') - - API_RATE_LIMIT_ENABLED = True - DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] + SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development + # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match + ADMIN_CLIENT_ID = 'notify-admin' + INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']} class Test(Development): - NOTIFY_EMAIL_DOMAIN = 'test.notify.com' FROM_NUMBER = 'testing' - NOTIFY_ENVIRONMENT = 'test' TESTING = True + ANTIVIRUS_ENABLED = True + DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] + + FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] + MMG_INBOUND_SMS_AUTH = ['testkey'] + MMG_INBOUND_SMS_USERNAME = ['username'] HIGH_VOLUME_SERVICE = [ '941b6f9a-50d7-4742-8d50-f365ca74bf27', @@ -443,137 +392,40 @@ class Test(Development): CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' CONTACT_LIST_BUCKET_NAME = 'test-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'test-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp' - # LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in CI - SQLALCHEMY_DATABASE_URI = os.getenv( - 'SQLALCHEMY_DATABASE_TEST_URI', - 'postgresql://postgres:chummy@db:5432/test_notification_api' - ) + SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI') CELERY = { **Config.CELERY, 'broker_url': 'you-forgot-to-mock-celery-in-your-tests://' } - ANTIVIRUS_ENABLED = True - - API_RATE_LIMIT_ENABLED = True - API_HOST_NAME = "http://localhost:6011" - - SMS_INBOUND_WHITELIST = ['203.0.113.195'] - FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999' - MMG_URL = 'https://example.com/mmg' - FIRETEXT_URL = 'https://example.com/firetext' - DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] - - -class Preview(Config): - NOTIFY_EMAIL_DOMAIN = 'notify.works' - NOTIFY_ENVIRONMENT = 'preview' - CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'preview-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'preview-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise' - FROM_NUMBER = 'preview' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = False - - -class Staging(Config): - NOTIFY_EMAIL_DOMAIN = 'staging-notify.works' - NOTIFY_ENVIRONMENT = 'staging' - CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'staging-contact-list' - # TEST_LETTERS_BUCKET_NAME = 'staging-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf' - # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' - # LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise' - FROM_NUMBER = 'stage' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = True - - -class Live(Config): - NOTIFY_ENVIRONMENT = 'live' +class Production(Config): # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get( - 'CSV_UPLOAD_BUCKET_NAME', - 'notifications-prototype-csv-upload' - ) # created in gsa sandbox + CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_BUCKET_NAME = os.environ.get( - 'CONTACT_LIST_BUCKET_NAME', - 'notifications-prototype-contact-list-upload' - ) # created in gsa sandbox + CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION') - # TODO: verify below buckets only used for letters - # TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox - # DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox - # LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox - # LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox - # INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox - # TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox - # LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox FROM_NUMBER = 'US Notify' - API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = True - SES_STUB_URL = None CRONITOR_ENABLED = True - # DEBUG = True - REDIS_ENABLED = os.environ.get('REDIS_ENABLED') - NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log') - - -class CloudFoundryConfig(Config): +class Staging(Production): pass -# CloudFoundry sandbox -class Sandbox(CloudFoundryConfig): - NOTIFY_EMAIL_DOMAIN = 'notify.works' - NOTIFY_ENVIRONMENT = 'sandbox' - CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list' - # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' - # TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters' - # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' - # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' - # LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan' - # INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf' - FROM_NUMBER = 'sandbox' - - configs = { 'development': Development, 'test': Test, - 'live': Live, - 'production': Live, 'staging': Staging, - 'preview': Preview, - 'sandbox': Sandbox + 'production': Production } diff --git a/sample.env b/sample.env index 3909b9beb..98309ede9 100644 --- a/sample.env +++ b/sample.env @@ -1,11 +1,11 @@ # STEPS TO SET UP -# +# # 1. Pull down AWS creds from cloud.gov using `cf env`, then update AWS section -# +# # 2. Uncomment either the Docker setup or the direct setup # # 3. Comment out the other setup -# +# # 4. Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) # # 5. Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values @@ -16,9 +16,9 @@ ############################################################# # AWS -AWS_REGION=us-west-2 AWS_ACCESS_KEY_ID="don't write secrets to the sample file" AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" +AWS_REGION=us-west-2 AWS_PINPOINT_REGION=us-west-2 AWS_US_TOLL_FREE_NUMBER=+18446120782 @@ -28,7 +28,6 @@ AWS_US_TOLL_FREE_NUMBER=+18446120782 ADMIN_BASE_URL=http://admin:6012 API_HOST_NAME=http://dev:6011 REDIS_URL=redis://redis:6380 -REDIS_ENABLED=1 SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api @@ -36,7 +35,6 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat # ADMIN_BASE_URL=http://localhost:6012 # API_HOST_NAME=http://localhost:6011 # REDIS_URL=redis://localhost:6379 -# REDIS_ENABLED=1 # SQLALCHEMY_DATABASE_URI=postgresql://localhost:5432/notification_api # SQLALCHEMY_DATABASE_TEST_URI=postgresql://localhost:5432/test_notification_api @@ -45,12 +43,12 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat # Debug DEBUG=True ANTIVIRUS_ENABLED=0 +REDIS_ENABLED=1 NOTIFY_ENVIRONMENT=development STATSD_HOST=localhost SES_STUB_URL=None NOTIFY_APP_NAME=api NOTIFY_EMAIL_DOMAIN=dispostable.com -NOTIFY_LOG_PATH=/workspace/logs/app.log ############################################################# @@ -58,9 +56,3 @@ NOTIFY_LOG_PATH=/workspace/logs/app.log FLASK_APP=application.py FLASK_ENV=development WERKZEUG_DEBUG_PIN=off -SECRET_KEY=dev-notify-secret-key -DANGEROUS_SALT=dev-notify-salt - -# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API -ADMIN_CLIENT_ID=notify-admin -ADMIN_CLIENT_SECRET=dev-notify-secret-key From 18c2fc4e16bb659a2351c603cb6b4b73d3086953 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 31 Oct 2022 14:27:52 -0400 Subject: [PATCH 06/12] add versions in pipfile for git deps --- Pipfile | 4 ++-- Pipfile.lock | 59 +++++++++++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Pipfile b/Pipfile index dea8f5f5c..3af486903 100644 --- a/Pipfile +++ b/Pipfile @@ -49,7 +49,7 @@ lxml = "==4.9.1" marshmallow = "==3.15.0" marshmallow-sqlalchemy = "==0.28.1" notifications-python-client = "==6.3.0" -notifications-utils = {git = "https://github.com/GSA/notifications-utils.git"} +notifications-utils = {version = "==56.0.3", git = "https://github.com/GSA/notifications-utils.git"} oscrypto = "==1.3.0" psycopg2-binary = "==2.9.3" pyjwt = "==2.4.0" @@ -60,7 +60,7 @@ werkzeug = "~=2.1.1" awscli-cwlogs = "==1.4.6" # gds metrics packages prometheus-client = "==0.14.1" -gds-metrics = {ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"} +gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"} [dev-packages] flake8 = "==4.0.1" diff --git a/Pipfile.lock b/Pipfile.lock index 3cdd1fea4..1faa1c1f1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4d2f356b612e2a1c813e6d0f0db1d73948a7124b19ff012d993b74ec5cc4b03e" + "sha256": "f4f8c5db036bc6412ed4651902b4d8d5906fc920063bf0182933f3947959d120" }, "pipfile-spec": 6, "requires": { @@ -383,7 +383,8 @@ }, "gds-metrics": { "git": "https://github.com/alphagov/gds_metrics_python.git", - "ref": "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72" + "ref": "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", + "version": "==0.2.4" }, "geojson": { "hashes": [ @@ -721,7 +722,8 @@ }, "notifications-utils": { "git": "https://github.com/GSA/notifications-utils.git", - "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd" + "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd", + "version": "==56.0.3" }, "orderedset": { "hashes": [ @@ -920,30 +922,31 @@ }, "pyrsistent": { "hashes": [ - "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", - "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc", - "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e", - "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26", - "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec", - "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286", - "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045", - "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec", - "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8", - "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c", - "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca", - "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22", - "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a", - "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96", - "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc", - "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1", - "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07", - "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6", - "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b", - "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5", - "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6" + "sha256:06579d46d8ad69529b28f88711191a7fe7103c92d04a9f338dc754f71b92efa0", + "sha256:1d0620474d509172e1c50b79d5626bfe1899f174bf650186a50c6ce31289ff52", + "sha256:2032d971711643049b4f2c3ca5155a855d507d73bad26dac8d4349e5c5dd6758", + "sha256:2c641111c3f110379bb9001dbb26b34eb8cafab3d0fa855dc161c391461a4aab", + "sha256:327f99800d04a9abcf580daecfd6dd4bfdb4a7e61c71bf2cd1189ef1ca44bade", + "sha256:39f15ad754384e744ac8b00805913bfa66c41131faaa3e4c45c4af0731f3e8f6", + "sha256:4c58bd93c4d502f52938fccdbe6c9d70df3a585c6b39d900fab5f76b604282aa", + "sha256:62a41037387ae849a493cd945e22b34d167a843d57f75b07dbfad6d96cef485c", + "sha256:62b704f18526a8fc243152de8f3f40ae39c5172baff10f50c0c5d5331d6f2342", + "sha256:6df99c3578dc4eb33f3eb26bc28277ab40a720b71649d940bff9c1f704377772", + "sha256:6ef7430e45c5fa0bb6c361cada4a08ed9c184b5ed086815a85c3bc8c5054566b", + "sha256:73b2db09fe15b6e444c0bd566a125a385ca6493456224ce8b367d734f079f576", + "sha256:73d4ec2997716af3c8f28f7e3d3a565d273a598982d2fe95639e07ce4db5da45", + "sha256:73e3e2fd9da009d558050697cc22ad689f89a14a2ef2e67304628a913e59c947", + "sha256:890f577aec554f142e01daf890221d10e4f93a9b1107998d631d3f075b55e8f8", + "sha256:8a34a2a8b220247658f7ced871197c390b3a6371d796a5869ab1c62abe0be527", + "sha256:8bc23e9ddcb523c3ffb4d712aa0bd5bc67b34ff4e2b23fb557012171bdb4013a", + "sha256:945297fc344fef4d540135180ce7babeb2291d124698cc6282f3eac624aa5e82", + "sha256:aaa869d9199d7d4c70a57678aff21654cc179c0c32bcfde87f1d65d0ff47e520", + "sha256:bc33fc20ddfd89b86b7710142963490d8c4ee8307ed6cc5e189a58fa72390eb9", + "sha256:cfe6d8b293d123255fd3b475b5f4e851eb5cbaee2064c8933aa27344381744ae", + "sha256:d16ac5ab3d9db78fed40c884d67079524e4cf8276639211ad9e6fa73e727727e" ], "markers": "python_version >= '3.7'", - "version": "==0.18.1" + "version": "==0.19.1" }, "python-dateutil": { "hashes": [ @@ -1813,11 +1816,11 @@ }, "pip-audit": { "hashes": [ - "sha256:a6205bb586f5964325b1af888914bf547d91f588a5f5b2c4d73f04a39fcc276f", - "sha256:cc7be2253f80dba44e8ae0002c26bf920114436b0097bfd483581c8b607caae2" + "sha256:6c7fd7c300559c99963f0b4be4543c55175e1550534228b8d27bce7dcd06ab34", + "sha256:be866655a365714c98782e00e64b092383ffb9171680aaaa0fb50ba75d775022" ], "index": "pypi", - "version": "==2.4.4" + "version": "==2.4.5" }, "pip-requirements-parser": { "hashes": [ From 41a52daca0059328973402e3ca2ddadfacc0042f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 31 Oct 2022 15:37:12 -0400 Subject: [PATCH 07/12] Clean up bucket settings --- app/aws/s3.py | 16 +++---- app/cloudfoundry_config.py | 61 +++++++++++---------------- app/config.py | 51 ++++++++++------------ manifest.yml | 7 +-- tests/app/test_cloudfoundry_config.py | 51 ++++++++++++++++------ tests/app/test_config.py | 56 ------------------------ 6 files changed, 93 insertions(+), 149 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 7312f4cbe..7a705c544 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -41,21 +41,21 @@ def file_exists( def get_job_location(service_id, job_id): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + current_app.config['CSV_UPLOAD_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, job_id), - current_app.config['CSV_UPLOAD_ACCESS_KEY'], - current_app.config['CSV_UPLOAD_SECRET_KEY'], - current_app.config['CSV_UPLOAD_REGION'], + current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'], + current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'], + current_app.config['CSV_UPLOAD_BUCKET']['region'], ) def get_contact_list_location(service_id, contact_list_id): return ( - current_app.config['CONTACT_LIST_BUCKET_NAME'], + current_app.config['CONTACT_LIST_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id), - current_app.config['CONTACT_LIST_ACCESS_KEY'], - current_app.config['CONTACT_LIST_SECRET_KEY'], - current_app.config['CONTACT_LIST_REGION'], + current_app.config['CONTACT_LIST_BUCKET']['access_key_id'], + current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'], + current_app.config['CONTACT_LIST_BUCKET']['region'], ) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 533f3d6af..1218e8852 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,41 +2,30 @@ import json import os -def find_by_service_name(services, service_name): - for i in range(len(services)): - if services[i]['name'] == service_name: - return services[i] - return None +class CloudfoundryConfig: + def __init__(self): + self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}') + buckets = self.parsed_services.get('s3') or [] + self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets} + self._empty_bucket_credentials = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + @property + def redis_url(self): + try: + return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace( + 'redis://', + 'rediss://' + ) + except KeyError: + return os.environ.get('REDIS_URL') + + def s3_credentials(self, service_name): + return self.s3_buckets.get(service_name) or self._empty_bucket_credentials -def extract_cloudfoundry_config(): - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - - # Postgres config - os.environ['SQLALCHEMY_DATABASE_URI'] = \ - vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql') - # Redis config - os.environ['REDIS_URL'] = \ - vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://') - - # CSV Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] - - # Contact List Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region'] +cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index 86f0be77b..723143c4f 100644 --- a/app/config.py +++ b/app/config.py @@ -5,12 +5,7 @@ from datetime import timedelta from celery.schedules import crontab from kombu import Exchange, Queue -if os.environ.get('VCAP_SERVICES'): - # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate - # standard environment variables from it - from app.cloudfoundry_config import extract_cloudfoundry_config - - extract_cloudfoundry_config() +from app.cloudfoundry_config import cloud_config class QueueNames(object): @@ -77,7 +72,8 @@ class Config(object): # Credentials # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API - ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID') + # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match + ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin') INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get( 'INTERNAL_CLIENT_API_KEYS', @@ -100,7 +96,7 @@ class Config(object): SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 - REDIS_URL = os.environ.get('REDIS_URL') + REDIS_URL = cloud_config.redis_url REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1' EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 @@ -350,27 +346,28 @@ class Config(object): DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') +def _default_s3_credentials(bucket_name): + return { + 'bucket': bucket_name, + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + + class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] # Buckets - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2') - CONTACT_LIST_BUCKET_NAME = 'local-contact-list' - CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2') + CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') # credential overrides DANGEROUS_SALT = 'dev-notify-salt' SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development - # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match - ADMIN_CLIENT_ID = 'notify-admin' - INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']} + INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']} class Test(Development): @@ -390,8 +387,8 @@ class Test(Development): '10d1b9c9-0072-4fa9-ae1c-595e333841da', ] - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'test-contact-list' + CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') # this is overriden in CI SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI') @@ -406,14 +403,10 @@ class Test(Development): class Production(Config): # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') - CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') - CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') - CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') - CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') - CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') - CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION') + CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}") + CONTACT_LIST_BUCKET = cloud_config.s3_credentials( + f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}") FROM_NUMBER = 'US Notify' CRONITOR_ENABLED = True diff --git a/manifest.yml b/manifest.yml index c07573fc6..77ee86a1c 100644 --- a/manifest.yml +++ b/manifest.yml @@ -32,16 +32,13 @@ applications: NOTIFY_LOG_PATH: /home/vcap/logs/app.log FLASK_APP: application.py FLASK_ENV: production - DEPLOY_ENV: ((env)) - NOTIFY_ENVIRONMENT: live + NOTIFY_ENVIRONMENT: ((env)) API_HOST_NAME: https://notifications-api.app.cloud.gov ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov - STATSD_HOST: localhost # Credentials variables INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' - ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) @@ -49,5 +46,3 @@ applications: AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 - - DVLA_EMAIL_ADDRESSES: [] diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 1d5db39d7..80fad2648 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -3,7 +3,14 @@ import os import pytest -from app.cloudfoundry_config import extract_cloudfoundry_config +from app.cloudfoundry_config import CloudfoundryConfig + +bucket_credentials = { + 'access_key_id': 'csv-access', + 'bucket': 'csv-upload-bucket', + 'region': 'us-gov-west-1', + 'secret_access_key': 'csv-secret' +} @pytest.fixture @@ -22,12 +29,7 @@ def vcap_services(): 's3': [ { 'name': 'notifications-api-csv-upload-bucket-test', - 'credentials': { - 'access_key_id': 'csv-access', - 'bucket': 'csv-upload-bucket', - 'region': 'us-gov-west-1', - 'secret_access_key': 'csv-secret' - } + 'credentials': bucket_credentials }, { 'name': 'notifications-api-contact-list-bucket-test', @@ -43,12 +45,33 @@ def vcap_services(): } -def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): - os.environ['DEPLOY_ENV'] = 'test' +def test_redis_url(vcap_services): os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) - extract_cloudfoundry_config() - assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri' - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' - assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' + assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379' + + +def test_redis_url_falls_back_to_REDIS_URL(): + expected = 'redis://yyy:6379' + os.environ['REDIS_URL'] = expected + os.environ['VCAP_SERVICES'] = "" + + assert CloudfoundryConfig().redis_url == expected + + +def test_s3_bucket_credentials(vcap_services): + os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) + + assert CloudfoundryConfig().s3_credentials('notifications-api-csv-upload-bucket-test') == bucket_credentials + + +def test_s3_bucket_credentials_falls_back_to_empty_creds(): + os.environ['VCAP_SERVICES'] = "" + expected = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + assert CloudfoundryConfig().s3_credentials('bucket') == expected diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 17bb96bb6..1f77278d1 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -1,62 +1,6 @@ -import importlib -import os -from unittest import mock - -import pytest - -from app import config from app.config import QueueNames -def cf_conf(): - os.environ['ADMIN_BASE_URL'] = 'cf' - - -@pytest.fixture -def reload_config(): - """ - Reset config, by simply re-running config.py from a fresh environment - """ - old_env = os.environ.copy() - - yield - - os.environ.clear() - for k, v in old_env.items(): - os.environ[k] = v - - importlib.reload(config) - - -def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ['VCAP_SERVICES'] = 'some json blob' - os.environ['VCAP_APPLICATION'] = 'some json blob' - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'cf' - assert config.Config.ADMIN_BASE_URL == 'cf' - - -def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ.pop('VCAP_SERVICES', None) - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert not cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'env' - assert config.Config.ADMIN_BASE_URL == 'env' - - def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues() From f7dce28546eb046b99539b1ab4a98b0d7a8037b3 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 31 Oct 2022 16:32:20 -0400 Subject: [PATCH 08/12] Use safe env getter for test verification --- tests/app/celery/test_ftp_update_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py index e0931d31f..4cc6c4baa 100644 --- a/tests/app/celery/test_ftp_update_tasks.py +++ b/tests/app/celery/test_ftp_update_tasks.py @@ -92,9 +92,9 @@ def test_update_letter_notifications_statuses_calls_with_correct_bucket_location s3_mock.assert_called_with('{}-ftp'.format( current_app.config['NOTIFY_EMAIL_DOMAIN']), 'NOTIFY-20170823160812-RSP.TXT', - os.environ['AWS_ACCESS_KEY_ID'], - os.environ['AWS_SECRET_ACCESS_KEY'], - os.environ['AWS_REGION'], + os.environ.get('AWS_ACCESS_KEY_ID'), + os.environ.get('AWS_SECRET_ACCESS_KEY'), + os.environ.get('AWS_REGION'), ) From f127a7c7aced570fee17d080bbaea3b44d5adf4d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 31 Oct 2022 16:44:03 -0400 Subject: [PATCH 09/12] =?UTF-8?q?add=20the=20requirements.txt=20file=20dur?= =?UTF-8?q?ing=20deploy=20=F0=9F=A4=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 3 + Pipfile.lock | 123 ++++++++++++++++------------------- 2 files changed, 59 insertions(+), 67 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 588b99c4f..dad470e5d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,6 +46,9 @@ jobs: - name: Install application dependencies run: make bootstrap + - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing + run: pipenv requirements > requirements.txt + - name: Deploy to cloud.gov uses: 18f/cg-deploy-action@main env: diff --git a/Pipfile.lock b/Pipfile.lock index 1faa1c1f1..883e22816 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -402,75 +402,64 @@ }, "greenlet": { "hashes": [ - "sha256:0120a879aa2b1ac5118bce959ea2492ba18783f65ea15821680a256dfad04754", - "sha256:025b8de2273d2809f027d347aa2541651d2e15d593bbce0d5f502ca438c54136", - "sha256:05ae7383f968bba4211b1fbfc90158f8e3da86804878442b4fb6c16ccbcaa519", - "sha256:0914f02fcaa8f84f13b2df4a81645d9e82de21ed95633765dd5cc4d3af9d7403", - "sha256:0971d37ae0eaf42344e8610d340aa0ad3d06cd2eee381891a10fe771879791f9", - "sha256:0a954002064ee919b444b19c1185e8cce307a1f20600f47d6f4b6d336972c809", - "sha256:0aa1845944e62f358d63fcc911ad3b415f585612946b8edc824825929b40e59e", - "sha256:104f29dd822be678ef6b16bf0035dcd43206a8a48668a6cae4d2fe9c7a7abdeb", - "sha256:11fc7692d95cc7a6a8447bb160d98671ab291e0a8ea90572d582d57361360f05", - "sha256:17a69967561269b691747e7f436d75a4def47e5efcbc3c573180fc828e176d80", - "sha256:2794eef1b04b5ba8948c72cc606aab62ac4b0c538b14806d9c0d88afd0576d6b", - "sha256:2c6e942ca9835c0b97814d14f78da453241837419e0d26f7403058e8db3e38f8", - "sha256:2ccdc818cc106cc238ff7eba0d71b9c77be868fdca31d6c3b1347a54c9b187b2", - "sha256:325f272eb997916b4a3fc1fea7313a8adb760934c2140ce13a2117e1b0a8095d", - "sha256:39464518a2abe9c505a727af7c0b4efff2cf242aa168be5f0daa47649f4d7ca8", - "sha256:3a24f3213579dc8459e485e333330a921f579543a5214dbc935bc0763474ece3", - "sha256:3aeac044c324c1a4027dca0cde550bd83a0c0fbff7ef2c98df9e718a5086c194", - "sha256:3c22998bfef3fcc1b15694818fc9b1b87c6cc8398198b96b6d355a7bcb8c934e", - "sha256:467b73ce5dcd89e381292fb4314aede9b12906c18fab903f995b86034d96d5c8", - "sha256:4a8b58232f5b72973350c2b917ea3df0bebd07c3c82a0a0e34775fc2c1f857e9", - "sha256:4f74aa0092602da2069df0bc6553919a15169d77bcdab52a21f8c5242898f519", - "sha256:5662492df0588a51d5690f6578f3bbbd803e7f8d99a99f3bf6128a401be9c269", - "sha256:5c2d21c2b768d8c86ad935e404cc78c30d53dea009609c3ef3a9d49970c864b5", - "sha256:5edf75e7fcfa9725064ae0d8407c849456553a181ebefedb7606bac19aa1478b", - "sha256:60839ab4ea7de6139a3be35b77e22e0398c270020050458b3d25db4c7c394df5", - "sha256:62723e7eb85fa52e536e516ee2ac91433c7bb60d51099293671815ff49ed1c21", - "sha256:64e10f303ea354500c927da5b59c3802196a07468332d292aef9ddaca08d03dd", - "sha256:66aa4e9a726b70bcbfcc446b7ba89c8cec40f405e51422c39f42dfa206a96a05", - "sha256:695d0d8b5ae42c800f1763c9fce9d7b94ae3b878919379150ee5ba458a460d57", - "sha256:70048d7b2c07c5eadf8393e6398595591df5f59a2f26abc2f81abca09610492f", - "sha256:7afa706510ab079fd6d039cc6e369d4535a48e202d042c32e2097f030a16450f", - "sha256:7cf37343e43404699d58808e51f347f57efd3010cc7cee134cdb9141bd1ad9ea", - "sha256:8149a6865b14c33be7ae760bcdb73548bb01e8e47ae15e013bf7ef9290ca309a", - "sha256:814f26b864ed2230d3a7efe0336f5766ad012f94aad6ba43a7c54ca88dd77cba", - "sha256:82a38d7d2077128a017094aff334e67e26194f46bd709f9dcdacbf3835d47ef5", - "sha256:83a7a6560df073ec9de2b7cb685b199dfd12519bc0020c62db9d1bb522f989fa", - "sha256:8415239c68b2ec9de10a5adf1130ee9cb0ebd3e19573c55ba160ff0ca809e012", - "sha256:88720794390002b0c8fa29e9602b395093a9a766b229a847e8d88349e418b28a", - "sha256:890f633dc8cb307761ec566bc0b4e350a93ddd77dc172839be122be12bae3e10", - "sha256:8926a78192b8b73c936f3e87929931455a6a6c6c385448a07b9f7d1072c19ff3", - "sha256:8c0581077cf2734569f3e500fab09c0ff6a2ab99b1afcacbad09b3c2843ae743", - "sha256:8fda1139d87ce5f7bd80e80e54f9f2c6fe2f47983f1a6f128c47bf310197deb6", - "sha256:91a84faf718e6f8b888ca63d0b2d6d185c8e2a198d2a7322d75c303e7097c8b7", - "sha256:924df1e7e5db27d19b1359dc7d052a917529c95ba5b8b62f4af611176da7c8ad", - "sha256:949c9061b8c6d3e6e439466a9be1e787208dec6246f4ec5fffe9677b4c19fcc3", - "sha256:9649891ab4153f217f319914455ccf0b86986b55fc0573ce803eb998ad7d6854", - "sha256:96656c5f7c95fc02c36d4f6ef32f4e94bb0b6b36e6a002c21c39785a4eec5f5d", - "sha256:a812df7282a8fc717eafd487fccc5ba40ea83bb5b13eb3c90c446d88dbdfd2be", - "sha256:a8d24eb5cb67996fb84633fdc96dbc04f2d8b12bfcb20ab3222d6be271616b67", - "sha256:bef49c07fcb411c942da6ee7d7ea37430f830c482bf6e4b72d92fd506dd3a427", - "sha256:bffba15cff4802ff493d6edcf20d7f94ab1c2aee7cfc1e1c7627c05f1102eee8", - "sha256:c0643250dd0756f4960633f5359884f609a234d4066686754e834073d84e9b51", - "sha256:c6f90234e4438062d6d09f7d667f79edcc7c5e354ba3a145ff98176f974b8132", - "sha256:c8c9301e3274276d3d20ab6335aa7c5d9e5da2009cccb01127bddb5c951f8870", - "sha256:c8ece5d1a99a2adcb38f69af2f07d96fb615415d32820108cd340361f590d128", - "sha256:cb863057bed786f6622982fb8b2c122c68e6e9eddccaa9fa98fd937e45ee6c4f", - "sha256:ccbe7129a282ec5797df0451ca1802f11578be018a32979131065565da89b392", - "sha256:d25cdedd72aa2271b984af54294e9527306966ec18963fd032cc851a725ddc1b", - "sha256:d75afcbb214d429dacdf75e03a1d6d6c5bd1fa9c35e360df8ea5b6270fb2211c", - "sha256:d7815e1519a8361c5ea2a7a5864945906f8e386fa1bc26797b4d443ab11a4589", - "sha256:eb6ac495dccb1520667cfea50d89e26f9ffb49fa28496dea2b95720d8b45eb54", - "sha256:ec615d2912b9ad807afd3be80bf32711c0ff9c2b00aa004a45fd5d5dde7853d9", - "sha256:f5e09dc5c6e1796969fd4b775ea1417d70e49a5df29aaa8e5d10675d9e11872c", - "sha256:f6661b58412879a2aa099abb26d3c93e91dedaba55a6394d1fb1512a77e85de9", - "sha256:f7d20c3267385236b4ce54575cc8e9f43e7673fc761b069c820097092e318e3b", - "sha256:fe7c51f8a2ab616cb34bc33d810c887e89117771028e1e3d3b77ca25ddeace04" + "sha256:069a8a557541a04518dc3beb9a78637e4e6b286814849a2ecfac529eaa78562b", + "sha256:089e123d80dbc6f61fff1ff0eae547b02c343d50968832716a7b0a33bea5f792", + "sha256:09f00f9938eb5ae1fe203558b56081feb0ca34a2895f8374cd01129ddf4d111c", + "sha256:0ba0f2e5c4a8f141952411e356dba05d6fe0c38325ee0e4f2d0c6f4c2c3263d5", + "sha256:0fa2a66fdf0d09929e79f786ad61529d4e752f452466f7ddaa5d03caf77a603d", + "sha256:1cfeae4dda32eb5c64df05d347c4496abfa57ad16a90082798a2bba143c6c854", + "sha256:20bf68672ae14ef2e2e6d3ac1f308834db1d0b920b3b0674eef48b2dce0498dd", + "sha256:2b8e1c939b363292ecc93999fb1ad53ffc5d0aac8e933e4362b62365241edda5", + "sha256:2be628bca0395610da08921f9376dd14317f37256d41078f5c618358467681e1", + "sha256:2f5d396a5457458460b0c28f738fc8ab2738ee61b00c3f845c7047a333acd96c", + "sha256:30198bccd774f9b6b1ba7564a0d02a79dd1fe926cfeb4107856fe16c9dfb441c", + "sha256:341053e0a96d512315c27c34fad4672c4573caf9eb98310c39e7747645c88d8b", + "sha256:3cc1abaf47cfcfdc9ac0bdff173cebab22cd54e9e3490135a4a9302d0ff3b163", + "sha256:3dc294afebf2acfd029373dbf3d01d36fd8d6888a03f5a006e2d690f66b153d9", + "sha256:49fcdd8ae391ffabb3b672397b58a9737aaff6b8cae0836e8db8ff386fcea802", + "sha256:4a1953465b7651073cffde74ed7d121e602ef9a9740d09ee137b01879ac15a2f", + "sha256:4be4dedbd2fa9b7c35627f322d6d3139cb125bc18d5ef2f40237990850ea446f", + "sha256:4c5ddadfe40e903c6217ed2b95a79f49e942bb98527547cc339fc7e43a424aad", + "sha256:5392ddb893e7fba237b988f846c4a80576557cc08664d56dc1a69c5c02bdc80c", + "sha256:6b28420ae290bfbf5d827f976abccc2f74f0a3f5e4fb69b66acf98f1cbe95e7e", + "sha256:6c66f0da8049ee3c126b762768179820d4c0ae0ca46ae489039e4da2fae39a52", + "sha256:6fd342126d825b76bf5b49717a7c682e31ed1114906cdec7f5a0c2ff1bc737a7", + "sha256:75c022803de010294366f3608d4bba3e346693b1b7427b79d57e3d924ed03838", + "sha256:79687c48e7f564be40c46b3afea6d141b8d66ffc2bc6147e026d491c6827954a", + "sha256:7acaa51355d5b9549d474dc71be6846ee9a8f2cb82f4936e5efa7a50bbeb94ad", + "sha256:8e8dbad9b4f4c3e37898914cfccb7c4f00dbe3146333cfe52a1a3103cc2ff97c", + "sha256:939963d0137ec92540d95b68b7f795e8dbadce0a1fca53e3e7ef8ddc18ee47cb", + "sha256:98b848a0b75e76b446dc71fdbac712d9078d96bb1c1607f049562dde1f8801e1", + "sha256:99e9851e40150504474915605649edcde259a4cd9bce2fcdeb4cf33ad0b5c293", + "sha256:9a4a9fea68fd98814999d91ea585e49ed68d7e199a70bef13a857439f60a4609", + "sha256:9d8dca31a39dd9f25641559b8cdf9066168c682dfcfbe0f797f03e4c9718a63a", + "sha256:9e5ead803b11b60b347e08e0f37234d9a595f44a6420026e47bcaf94190c3cd6", + "sha256:a245898ec5e9ca0bc87a63e4e222cc633dc4d1f1a0769c34a625ad67edb9f9de", + "sha256:a65205e6778142528978b4acca76888e7e7f0be261e395664e49a5c21baa2141", + "sha256:aa2b371c3633e694d043d6cec7376cb0031c6f67029f37eef40bda105fd58753", + "sha256:adcf45221f253b3a681c99da46fa6ac33596fa94c2f30c54368f7ee1c4563a39", + "sha256:b4fd73b62c1038e7ee938b1de328eaa918f76aa69c812beda3aff8a165494201", + "sha256:b89b78ffb516c2921aa180c2794082666e26680eef05996b91f46127da24d964", + "sha256:bc283f99a4815ef70cad537110e3e03abcef56ab7d005ba9a8c6ec33054ce9c0", + "sha256:c1e93ef863810fba75faf418f0861dbf59bfe01a7b5d0a91d39603df58d3d3fa", + "sha256:c3aa7d3bc545162a6676445709b24a2a375284dc5e2f2432d58b80827c2bd91c", + "sha256:c8c67ecda450ad4eac7837057f5deb96effa836dacaf04747710ccf8eeb73092", + "sha256:cc211c2ff5d3b2ba8d557a71e3b4f0f0a2020067515143a9516ea43884271192", + "sha256:d4e7642366e638f45d70c5111590a56fbd0ffb7f474af20c6c67c01270bcf5cf", + "sha256:d58d4b4dc82e2d21ebb7dd7d3a6d370693b2236a1407fe3988dc1d4ea07575f9", + "sha256:d65d7d1ff64fb300127d2ffd27db909de4d21712a5dde59a3ad241fb65ee83d7", + "sha256:d71feebf5c8041c80dfda76427e14e3ca00bca042481bd3e9612a9d57b2cbbf7", + "sha256:d8bacecee0c9348ab7c95df810e12585e9e8c331dfc1e22da4ed0bd635a5f483", + "sha256:e0d7efab8418c1fb3ea00c4abb89e7b0179a952d0d53ad5fcff798ca7440f8e8", + "sha256:e7a0dca752b4e3395890ab4085c3ec3838d73714261914c01b53ed7ea23b5867", + "sha256:e7ec3f2465ba9b7d25895307abe1c1c101a257c54b9ea1522bbcbe8ca8793735", + "sha256:eca9c0473de053dcc92156dd62c38c3578628b536c7f0cd66e655e211c14ac32", + "sha256:efdbbbf7b6c8d5be52977afa65b9bb7b658bab570543280e76c0fabc647175ed", + "sha256:f7edbd2957f72aea357241fe42ffc712a8e9b8c2c42f24e2ef5d97b255f66172", + "sha256:f8a10e14238407be3978fa6d190eb3724f9d766655fefc0134fd5482f1fb0108" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.1.3.post0" + "version": "==2.0.0" }, "gunicorn": { "extras": [ From 5ff4cfe7e3d8bfad6a845a33246b119b32018415 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 1 Nov 2022 09:35:39 -0400 Subject: [PATCH 10/12] Clarify good value of NOTIFY_EMAIL_DOMAIN in development --- sample.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample.env b/sample.env index 98309ede9..ca51a6007 100644 --- a/sample.env +++ b/sample.env @@ -6,7 +6,7 @@ # # 3. Comment out the other setup # -# 4. Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) +# 4. If needed, set `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) # # 5. Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values # @@ -48,7 +48,7 @@ NOTIFY_ENVIRONMENT=development STATSD_HOST=localhost SES_STUB_URL=None NOTIFY_APP_NAME=api -NOTIFY_EMAIL_DOMAIN=dispostable.com +# NOTIFY_EMAIL_DOMAIN=notify.sandbox.10x.gsa.gov ############################################################# From a4aedee572d7e865ac64ca1d0d7a14e83cc6aa7f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 1 Nov 2022 09:35:55 -0400 Subject: [PATCH 11/12] Add Procfile.dev to ease running of flask & celery in dev --- Procfile.dev | 2 ++ README.md | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Procfile.dev diff --git a/Procfile.dev b/Procfile.dev new file mode 100644 index 000000000..bc4af912c --- /dev/null +++ b/Procfile.dev @@ -0,0 +1,2 @@ +web: make run-flask +worker: make run-celery diff --git a/README.md b/README.md index 79a682726..43a00cbf0 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,10 @@ Our other repositories are: `make run-celery` +1. Or, run flask and celery together with [foreman](https://rubygems.org/gems/foreman) + + `foreman start -f Procfile.dev` + ### VS Code && Docker installation @@ -97,4 +101,3 @@ If you're working in VS Code, you can also leverage Docker for a containerized d `make run-celery` NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container. - From db96c4d82f7be1779ead1fa0011fc97154378584 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 1 Nov 2022 09:54:31 -0400 Subject: [PATCH 12/12] Replace foreman with honcho --- Makefile | 4 ++++ Pipfile | 1 + Pipfile.lock | 22 +++++++++++++++------- README.md | 14 ++++++++------ 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b14f342fa..9ced0aaa5 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ bootstrap: ## Set up everything to run the app bootstrap-with-docker: ## Build the image to run the app in Docker docker build -f docker/Dockerfile -t notifications-api . +.PHONY: run-procfile +run-procfile: + pipenv run honcho start -f Procfile.dev + .PHONY: run-flask run-flask: ## Run flask pipenv run flask run -p 6011 --host=0.0.0.0 diff --git a/Pipfile b/Pipfile index 3af486903..0ae471394 100644 --- a/Pipfile +++ b/Pipfile @@ -77,6 +77,7 @@ requests-mock = "==1.9.3" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} pip-audit = "*" bandit = "*" +honcho = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 883e22816..53ac2cf4d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f4f8c5db036bc6412ed4651902b4d8d5906fc920063bf0182933f3947959d120" + "sha256": "b1e61b613cb208ab42359fe335224367647de3b568adf4ed0872976d738a054b" }, "pipfile-spec": 6, "requires": { @@ -963,10 +963,10 @@ }, "pytz": { "hashes": [ - "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", - "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427", + "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2" ], - "version": "==2022.5" + "version": "==2022.6" }, "pyyaml": { "hashes": [ @@ -1581,6 +1581,14 @@ "markers": "python_version >= '3.7'", "version": "==3.1.29" }, + "honcho": { + "hashes": [ + "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f", + "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9" + ], + "index": "pypi", + "version": "==1.1.0" + }, "html5lib": { "hashes": [ "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", @@ -1931,10 +1939,10 @@ }, "pytz": { "hashes": [ - "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", - "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427", + "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2" ], - "version": "==2022.5" + "version": "==2022.6" }, "pyyaml": { "hashes": [ diff --git a/README.md b/README.md index ad7bee76b..b8cd421c0 100644 --- a/README.md +++ b/README.md @@ -58,17 +58,19 @@ Our other repositories are: # follow the instructions in .env ``` -1. Run Flask +1. Run the web server and background worker - `make run-flask` + `make run-procfile` -1. Run Celery +1. Or run them individually: - `make run-celery` + * Run Flask (web server) -1. Or, run flask and celery together with [foreman](https://rubygems.org/gems/foreman) + `make run-flask` - `foreman start -f Procfile.dev` + * Run Celery (background worker) + + `make run-celery` ### VS Code && Docker installation