diff --git a/.gitignore b/.gitignore index 57c37a6f7..220a2d3b2 100644 --- a/.gitignore +++ b/.gitignore @@ -78,5 +78,6 @@ celerybeat-schedule # CloudFoundry .cf varsfile* +.secret* /scripts/run_my_tests.sh diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index d21ca5310..000000000 --- a/.secrets.baseline +++ /dev/null @@ -1,378 +0,0 @@ -{ - "version": "1.2.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - } - ], - "results": { - "README.md": [ - { - "type": "Secret Keyword", - "filename": "README.md", - "hashed_secret": "3bf710d852e4ffb2bf157dccd3e6099d9cd6654f", - "is_verified": false, - "line_number": 30 - }, - { - "type": "Secret Keyword", - "filename": "README.md", - "hashed_secret": "fc5462be70401cb2f7940d506ff413e722bb9bbd", - "is_verified": false, - "line_number": 31 - } - ], - "app/config.py": [ - { - "type": "Secret Keyword", - "filename": "app/config.py", - "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", - "is_verified": false, - "line_number": 432 - } - ], - "app/models.py": [ - { - "type": "Secret Keyword", - "filename": "app/models.py", - "hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3", - "is_verified": false, - "line_number": 1891 - } - ], - "docker-compose.devcontainer.yml": [ - { - "type": "Secret Keyword", - "filename": "docker-compose.devcontainer.yml", - "hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4", - "is_verified": false, - "line_number": 12 - } - ], - "migrations/versions/0025_notify_service_data.py": [ - { - "type": "Secret Keyword", - "filename": "migrations/versions/0025_notify_service_data.py", - "hashed_secret": "14153c7b93d1070d1579eb85d81cf660b330befc", - "is_verified": false, - "line_number": 83 - } - ], - "pytest.ini": [ - { - "type": "Secret Keyword", - "filename": "pytest.ini", - "hashed_secret": "e501a39b67f82817f7ec580a36f6932e5377e59c", - "is_verified": false, - "line_number": 5 - }, - { - "type": "Secret Keyword", - "filename": "pytest.ini", - "hashed_secret": "a3e8fada29009777b0855fa5fadeb0bb91de4ef6", - "is_verified": false, - "line_number": 6 - } - ], - "sample.env": [ - { - "type": "Basic Auth Credentials", - "filename": "sample.env", - "hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4", - "is_verified": false, - "line_number": 33 - } - ], - "tests/app/authentication/test_authentication.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/authentication/test_authentication.py", - "hashed_secret": "9ba831ebe84a351453e3bd542a5e692316120170", - "is_verified": false, - "line_number": 42 - }, - { - "type": "Secret Keyword", - "filename": "tests/app/authentication/test_authentication.py", - "hashed_secret": "23332b8380ee6a1c573083631c41d8d5130e8d2a", - "is_verified": false, - "line_number": 186 - } - ], - "tests/app/aws/test_s3.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/app/aws/test_s3.py", - "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747", - "is_verified": false, - "line_number": 13 - } - ], - "tests/app/clients/test_cbc_proxy.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/clients/test_cbc_proxy.py", - "hashed_secret": "ba59e920f3d2ebb6a219a8b7e40ecb606ea697dd", - "is_verified": false, - "line_number": 39 - } - ], - "tests/app/clients/test_document_download.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/clients/test_document_download.py", - "hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f", - "is_verified": false, - "line_number": 16 - } - ], - "tests/app/conftest.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/conftest.py", - "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", - "is_verified": false, - "line_number": 620 - }, - { - "type": "Secret Keyword", - "filename": "tests/app/conftest.py", - "hashed_secret": "4ff75f80957469c4b6af5824cb99bf4919abad98", - "is_verified": false, - "line_number": 621 - } - ], - "tests/app/dao/test_services_dao.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/dao/test_services_dao.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 259 - } - ], - "tests/app/dao/test_users_dao.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/dao/test_users_dao.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 49 - }, - { - "type": "Secret Keyword", - "filename": "tests/app/dao/test_users_dao.py", - "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba", - "is_verified": false, - "line_number": 164 - } - ], - "tests/app/db.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/db.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 93 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/app/db.py", - "hashed_secret": "2020f1df1bb8bbe6d7f3e6b2d136cd2a38ad7fed", - "is_verified": false, - "line_number": 891 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/app/db.py", - "hashed_secret": "0362114f115be22057673eda269208449bec65ff", - "is_verified": false, - "line_number": 892 - } - ], - "tests/app/letters/test_returned_letters.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/app/letters/test_returned_letters.py", - "hashed_secret": "d782ccb6785b5e3cef91a149f946a38cea88fb81", - "is_verified": false, - "line_number": 5 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/app/letters/test_returned_letters.py", - "hashed_secret": "52f1e518a8f64e70f1f443eebe7cdd2e8cdaf83f", - "is_verified": false, - "line_number": 7 - } - ], - "tests/app/notifications/test_receive_notification.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/notifications/test_receive_notification.py", - "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6", - "is_verified": false, - "line_number": 25 - } - ], - "tests/app/notifications/test_validators.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/notifications/test_validators.py", - "hashed_secret": "9c2a6e4809aeef7b7712ca4db05a681452f4f748", - "is_verified": false, - "line_number": 401 - } - ], - "tests/app/service/test_rest.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/service/test_rest.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 1297 - } - ], - "tests/app/template/test_rest.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/template/test_rest.py", - "hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f", - "is_verified": false, - "line_number": 1168 - } - ], - "tests/app/user/test_rest.py": [ - { - "type": "Secret Keyword", - "filename": "tests/app/user/test_rest.py", - "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", - "is_verified": false, - "line_number": 108 - }, - { - "type": "Secret Keyword", - "filename": "tests/app/user/test_rest.py", - "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", - "is_verified": false, - "line_number": 889 - } - ], - "tests/app/v2/broadcast/test_post_broadcast.py": [ - { - "type": "Hex High Entropy String", - "filename": "tests/app/v2/broadcast/test_post_broadcast.py", - "hashed_secret": "afddbc89a4e7b99f99eb0b1fad44d7aed566c77f", - "is_verified": false, - "line_number": 99 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/app/v2/broadcast/test_post_broadcast.py", - "hashed_secret": "a1f83866aff48443fffba0f8c6f11da4ad2e6763", - "is_verified": false, - "line_number": 215 - } - ] - }, - "generated_at": "2022-06-27T21:40:11Z" -} diff --git a/app/config.py b/app/config.py index d117ddda0..f5f7f9325 100644 --- a/app/config.py +++ b/app/config.py @@ -103,9 +103,6 @@ class Config(object): # DB conection string SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI') - # Redis conection string - REDIS_URL = os.getenv('REDIS_URL') - # AWS SMS AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2") AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782") @@ -122,7 +119,7 @@ class Config(object): # URL of redis instance REDIS_URL = os.getenv('REDIS_URL') - REDIS_ENABLED = True + REDIS_ENABLED = 1 EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 @@ -548,8 +545,13 @@ class Live(Config): API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True SES_STUB_URL = None - CRONITOR_ENABLED = True + + # DEBUG = True + REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + + NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH', 'application.log') + REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0') class CloudFoundryConfig(Config): diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json index cf46cba59..3126c6d12 100644 --- a/devcontainer-api/.devcontainer.json +++ b/devcontainer-api/.devcontainer.json @@ -13,7 +13,8 @@ }, "python.linting.enabled": true, "python.linting.pylintEnabled": true, - "python.pythonPath": "/usr/local/bin/python" + "python.pythonPath": "/usr/local/bin/python", + "python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint" }, "features": { "docker-from-docker": { @@ -22,7 +23,7 @@ } }, "extensions": [ - "ms-python.black-formatter", + "ms-python.python", "donjayamanne.python-extension-pack", "ms-azuretools.vscode-docker", "ms-python.vscode-pylance", @@ -35,7 +36,7 @@ "GitHub.copilot", "ms-vsliveshare.vsliveshare", "mtxr.sqltools", - "mtxr.sqltools-driver-pg", + "mtxr.sqltools-driver-pg" ], "postCreateCommand": "notify-dev-entrypoint.sh", "remoteUser": "vscode" diff --git a/devcontainer-api/scripts/notify-dev-entrypoint.sh b/devcontainer-api/scripts/notify-dev-entrypoint.sh index 7ae62f240..bd7b7a30b 100755 --- a/devcontainer-api/scripts/notify-dev-entrypoint.sh +++ b/devcontainer-api/scripts/notify-dev-entrypoint.sh @@ -23,7 +23,7 @@ echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc # echo -e "source <(kubectl completion zsh)" >> ~/.zshrc # echo -e "complete -F __start_kubectl k" >> ~/.zshrc -cd /workspace +cd /workspace # Warm up git index prior to display status in prompt else it will # be quite slow on every invocation of starship. diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 000000000..6ec88245c Binary files /dev/null and b/dump.rdb differ diff --git a/logs/app.log.json b/logs/app.log.json new file mode 100644 index 000000000..2ac3ecd7b --- /dev/null +++ b/logs/app.log.json @@ -0,0 +1,25 @@ +{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:41:53", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:42:30", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} +{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:02", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} +{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:05", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:59:55", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:38", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"} +{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:52", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} +{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:56", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Email already registered for user 7796f6d5-4fba-48d5-bfa1-80c5b3465d24", "pathname": "/workspace/app/user/rest.py", "lineno": 415, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "To email is testreceiver@dispostable.com", "pathname": "/workspace/app/user/rest.py", "lineno": 418, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "template.id is 0880fbb1-a0c6-46f0-9a8e-36c986381ceb", "pathname": "/workspace/app/user/rest.py", "lineno": 423, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "service.id is d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/user/rest.py", "lineno": 424, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Presisting notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 108, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Presisting notification with id 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 114, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Presisting notification with to address: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 139, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Presisting notification with type: email", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 149, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Presisting notification to formatted email: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 151, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Firing dao_create_notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 159, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Redis enabled, querying cache key for service id: d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 162, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key: d6aa2c68-a2d9-4437-ab19-3ae8eb202553-2022-06-29-count", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 164, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key does not exist", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 166, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Set redis daily limit cache key to 1", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 172, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "email 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4 created at 2022-06-29 06:07:57.373123", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 177, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Sending notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 442, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} +{"name": "app", "levelname": "INFO", "message": "Sent notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 446, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"} diff --git a/sample.env b/sample.env index fec9cba17..9e9ccedd1 100644 --- a/sample.env +++ b/sample.env @@ -1,3 +1,5 @@ +### REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### + # Debug DEBUG=True ANTIVIRUS_ENABLED=0 @@ -7,6 +9,7 @@ STATSD_HOST=localhost SES_STUB_URL=None NOTIFY_APP_NAME=api NOTIFY_EMAIL_DOMAIN=dispostable.com +NOTIFY_LOG_PATH=/workspace/logs/app.log # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID=notify-admin