Merge pull request #15 from 18F/jim/062822/logging

Fix Redis, Improve Logging
This commit is contained in:
Christa Hartsock
2022-07-07 15:37:32 -07:00
committed by GitHub
8 changed files with 41 additions and 387 deletions

1
.gitignore vendored
View File

@@ -78,5 +78,6 @@ celerybeat-schedule
# CloudFoundry
.cf
varsfile*
.secret*
/scripts/run_my_tests.sh

View File

@@ -1,378 +0,0 @@
{
"version": "1.2.0",
"plugins_used": [
{
"name": "ArtifactoryDetector"
},
{
"name": "AWSKeyDetector"
},
{
"name": "AzureStorageKeyDetector"
},
{
"name": "Base64HighEntropyString",
"limit": 4.5
},
{
"name": "BasicAuthDetector"
},
{
"name": "CloudantDetector"
},
{
"name": "GitHubTokenDetector"
},
{
"name": "HexHighEntropyString",
"limit": 3.0
},
{
"name": "IbmCloudIamDetector"
},
{
"name": "IbmCosHmacDetector"
},
{
"name": "JwtTokenDetector"
},
{
"name": "KeywordDetector",
"keyword_exclude": ""
},
{
"name": "MailchimpDetector"
},
{
"name": "NpmDetector"
},
{
"name": "PrivateKeyDetector"
},
{
"name": "SendGridDetector"
},
{
"name": "SlackDetector"
},
{
"name": "SoftlayerDetector"
},
{
"name": "SquareOAuthDetector"
},
{
"name": "StripeDetector"
},
{
"name": "TwilioKeyDetector"
}
],
"filters_used": [
{
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
},
{
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
"min_level": 2
},
{
"path": "detect_secrets.filters.heuristic.is_indirect_reference"
},
{
"path": "detect_secrets.filters.heuristic.is_likely_id_string"
},
{
"path": "detect_secrets.filters.heuristic.is_lock_file"
},
{
"path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
},
{
"path": "detect_secrets.filters.heuristic.is_potential_uuid"
},
{
"path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
},
{
"path": "detect_secrets.filters.heuristic.is_sequential_string"
},
{
"path": "detect_secrets.filters.heuristic.is_swagger_file"
},
{
"path": "detect_secrets.filters.heuristic.is_templated_secret"
}
],
"results": {
"README.md": [
{
"type": "Secret Keyword",
"filename": "README.md",
"hashed_secret": "3bf710d852e4ffb2bf157dccd3e6099d9cd6654f",
"is_verified": false,
"line_number": 30
},
{
"type": "Secret Keyword",
"filename": "README.md",
"hashed_secret": "fc5462be70401cb2f7940d506ff413e722bb9bbd",
"is_verified": false,
"line_number": 31
}
],
"app/config.py": [
{
"type": "Secret Keyword",
"filename": "app/config.py",
"hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
"is_verified": false,
"line_number": 432
}
],
"app/models.py": [
{
"type": "Secret Keyword",
"filename": "app/models.py",
"hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3",
"is_verified": false,
"line_number": 1891
}
],
"docker-compose.devcontainer.yml": [
{
"type": "Secret Keyword",
"filename": "docker-compose.devcontainer.yml",
"hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
"is_verified": false,
"line_number": 12
}
],
"migrations/versions/0025_notify_service_data.py": [
{
"type": "Secret Keyword",
"filename": "migrations/versions/0025_notify_service_data.py",
"hashed_secret": "14153c7b93d1070d1579eb85d81cf660b330befc",
"is_verified": false,
"line_number": 83
}
],
"pytest.ini": [
{
"type": "Secret Keyword",
"filename": "pytest.ini",
"hashed_secret": "e501a39b67f82817f7ec580a36f6932e5377e59c",
"is_verified": false,
"line_number": 5
},
{
"type": "Secret Keyword",
"filename": "pytest.ini",
"hashed_secret": "a3e8fada29009777b0855fa5fadeb0bb91de4ef6",
"is_verified": false,
"line_number": 6
}
],
"sample.env": [
{
"type": "Basic Auth Credentials",
"filename": "sample.env",
"hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
"is_verified": false,
"line_number": 33
}
],
"tests/app/authentication/test_authentication.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/authentication/test_authentication.py",
"hashed_secret": "9ba831ebe84a351453e3bd542a5e692316120170",
"is_verified": false,
"line_number": 42
},
{
"type": "Secret Keyword",
"filename": "tests/app/authentication/test_authentication.py",
"hashed_secret": "23332b8380ee6a1c573083631c41d8d5130e8d2a",
"is_verified": false,
"line_number": 186
}
],
"tests/app/aws/test_s3.py": [
{
"type": "Hex High Entropy String",
"filename": "tests/app/aws/test_s3.py",
"hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
"is_verified": false,
"line_number": 13
}
],
"tests/app/clients/test_cbc_proxy.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/clients/test_cbc_proxy.py",
"hashed_secret": "ba59e920f3d2ebb6a219a8b7e40ecb606ea697dd",
"is_verified": false,
"line_number": 39
}
],
"tests/app/clients/test_document_download.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/clients/test_document_download.py",
"hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
"is_verified": false,
"line_number": 16
}
],
"tests/app/conftest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/conftest.py",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 620
},
{
"type": "Secret Keyword",
"filename": "tests/app/conftest.py",
"hashed_secret": "4ff75f80957469c4b6af5824cb99bf4919abad98",
"is_verified": false,
"line_number": 621
}
],
"tests/app/dao/test_services_dao.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_services_dao.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 259
}
],
"tests/app/dao/test_users_dao.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_users_dao.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 49
},
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_users_dao.py",
"hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
"is_verified": false,
"line_number": 164
}
],
"tests/app/db.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/db.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 93
},
{
"type": "Base64 High Entropy String",
"filename": "tests/app/db.py",
"hashed_secret": "2020f1df1bb8bbe6d7f3e6b2d136cd2a38ad7fed",
"is_verified": false,
"line_number": 891
},
{
"type": "Base64 High Entropy String",
"filename": "tests/app/db.py",
"hashed_secret": "0362114f115be22057673eda269208449bec65ff",
"is_verified": false,
"line_number": 892
}
],
"tests/app/letters/test_returned_letters.py": [
{
"type": "Hex High Entropy String",
"filename": "tests/app/letters/test_returned_letters.py",
"hashed_secret": "d782ccb6785b5e3cef91a149f946a38cea88fb81",
"is_verified": false,
"line_number": 5
},
{
"type": "Hex High Entropy String",
"filename": "tests/app/letters/test_returned_letters.py",
"hashed_secret": "52f1e518a8f64e70f1f443eebe7cdd2e8cdaf83f",
"is_verified": false,
"line_number": 7
}
],
"tests/app/notifications/test_receive_notification.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/notifications/test_receive_notification.py",
"hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
"is_verified": false,
"line_number": 25
}
],
"tests/app/notifications/test_validators.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/notifications/test_validators.py",
"hashed_secret": "9c2a6e4809aeef7b7712ca4db05a681452f4f748",
"is_verified": false,
"line_number": 401
}
],
"tests/app/service/test_rest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/service/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 1297
}
],
"tests/app/template/test_rest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/template/test_rest.py",
"hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
"is_verified": false,
"line_number": 1168
}
],
"tests/app/user/test_rest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 108
},
{
"type": "Secret Keyword",
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 889
}
],
"tests/app/v2/broadcast/test_post_broadcast.py": [
{
"type": "Hex High Entropy String",
"filename": "tests/app/v2/broadcast/test_post_broadcast.py",
"hashed_secret": "afddbc89a4e7b99f99eb0b1fad44d7aed566c77f",
"is_verified": false,
"line_number": 99
},
{
"type": "Hex High Entropy String",
"filename": "tests/app/v2/broadcast/test_post_broadcast.py",
"hashed_secret": "a1f83866aff48443fffba0f8c6f11da4ad2e6763",
"is_verified": false,
"line_number": 215
}
]
},
"generated_at": "2022-06-27T21:40:11Z"
}

View File

@@ -103,9 +103,6 @@ class Config(object):
# DB conection string
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
# Redis conection string
REDIS_URL = os.getenv('REDIS_URL')
# AWS SMS
AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2")
AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782")
@@ -122,7 +119,7 @@ class Config(object):
# URL of redis instance
REDIS_URL = os.getenv('REDIS_URL')
REDIS_ENABLED = True
REDIS_ENABLED = 1
EXPIRE_CACHE_TEN_MINUTES = 600
EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
@@ -548,9 +545,14 @@ class Live(Config):
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = True
SES_STUB_URL = None
CRONITOR_ENABLED = True
# DEBUG = True
REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH', 'application.log')
REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
class CloudFoundryConfig(Config):
pass

View File

@@ -13,7 +13,8 @@
},
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.pythonPath": "/usr/local/bin/python"
"python.pythonPath": "/usr/local/bin/python",
"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint"
},
"features": {
"docker-from-docker": {
@@ -22,7 +23,7 @@
}
},
"extensions": [
"ms-python.black-formatter",
"ms-python.python",
"donjayamanne.python-extension-pack",
"ms-azuretools.vscode-docker",
"ms-python.vscode-pylance",
@@ -35,7 +36,7 @@
"GitHub.copilot",
"ms-vsliveshare.vsliveshare",
"mtxr.sqltools",
"mtxr.sqltools-driver-pg",
"mtxr.sqltools-driver-pg"
],
"postCreateCommand": "notify-dev-entrypoint.sh",
"remoteUser": "vscode"

BIN
dump.rdb Normal file

Binary file not shown.

25
logs/app.log.json Normal file
View File

@@ -0,0 +1,25 @@
{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:41:53", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:42:30", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"}
{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:02", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"}
{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:43:05", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T05:59:55", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:38", "requestId": "no-request-id", "application": "app", "service_id": "no-service-id", "logType": "application"}
{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:52", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"}
{"name": "delivery", "levelname": "INFO", "message": "Logging configured", "pathname": "/home/vscode/.local/lib/python3.9/site-packages/notifications_utils/logging.py", "lineno": 37, "time": "2022-06-29T06:00:56", "requestId": "no-request-id", "application": "delivery", "service_id": "no-service-id", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Email already registered for user 7796f6d5-4fba-48d5-bfa1-80c5b3465d24", "pathname": "/workspace/app/user/rest.py", "lineno": 415, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "To email is testreceiver@dispostable.com", "pathname": "/workspace/app/user/rest.py", "lineno": 418, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "template.id is 0880fbb1-a0c6-46f0-9a8e-36c986381ceb", "pathname": "/workspace/app/user/rest.py", "lineno": 423, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "service.id is d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/user/rest.py", "lineno": 424, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Presisting notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 108, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Presisting notification with id 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 114, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Presisting notification with to address: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 139, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Presisting notification with type: email", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 149, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Presisting notification to formatted email: testreceiver@dispostable.com", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 151, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Firing dao_create_notification", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 159, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Redis enabled, querying cache key for service id: d6aa2c68-a2d9-4437-ab19-3ae8eb202553", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 162, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key: d6aa2c68-a2d9-4437-ab19-3ae8eb202553-2022-06-29-count", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 164, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Redis daily limit cache key does not exist", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 166, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Set redis daily limit cache key to 1", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 172, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "email 6bc43c9a-79d2-4b6e-b9cf-4fb6ec5a10c4 created at 2022-06-29 06:07:57.373123", "pathname": "/workspace/app/notifications/process_notifications.py", "lineno": 177, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Sending notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 442, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}
{"name": "app", "levelname": "INFO", "message": "Sent notification to queue", "pathname": "/workspace/app/user/rest.py", "lineno": 446, "time": "2022-06-29T06:07:57", "requestId": null, "application": "app", "service_id": "notify-admin", "logType": "application"}

View File

@@ -1,3 +1,5 @@
### REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ###
# Debug
DEBUG=True
ANTIVIRUS_ENABLED=0
@@ -7,6 +9,7 @@ STATSD_HOST=localhost
SES_STUB_URL=None
NOTIFY_APP_NAME=api
NOTIFY_EMAIL_DOMAIN=dispostable.com
NOTIFY_LOG_PATH=/workspace/logs/app.log
# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
ADMIN_CLIENT_ID=notify-admin