From f539cad4b4a831a7c1a75c6208cfb5da72c91dd5 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 11:55:26 -0400 Subject: [PATCH 01/11] Added some logging up front when the app is initialized to see if DANGEROUS_SALT and SECRET_KEY are both available and have data in them. Signed-off-by: Cliff Hill --- app/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index e99a5ae51..be017763c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -233,6 +233,14 @@ def create_app(application): ) logging.init_app(application) + import hashlib + + for key in ("SECRET_KEY", "DANGEROUS_SALT"): + application.logger.info( + f"{key} Length: {len(application.config[key])}; " + f"SHA-1: {hashlib.sha1(application.config[key].encode('utf8'), usedforsecurity=False).hexdigest()}" + ) + login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default" login_manager.session_protection = None From ec9a36e0dd01819dd6e28e86302f256b352c0535 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 15:09:28 -0400 Subject: [PATCH 02/11] Activating CSRF for staging/demo/dev environments. Signed-off-by: Cliff Hill --- app/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config.py b/app/config.py index 77138ca16..8922506f1 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = True ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -164,7 +164,7 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = True # buckets - mirror staging CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( From 9e94b5a242cc3971c4cb1af1f0e6318008fbd3c0 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 16:10:18 -0400 Subject: [PATCH 03/11] saabling CSRF in dev because it breaks all the tests. Signed-off-by: Cliff Hill --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 8922506f1..fc358549d 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = True + WTF_CSRF_ENABLED = False ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" From 530206e33a1dd19941e5405b84e1edc7e7c16ed5 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 10:13:15 -0400 Subject: [PATCH 04/11] Cleaning up the logging, gave some additional notes, removed hashing. Signed-off-by: Cliff Hill --- app/__init__.py | 19 +++++++++++++------ app/config.py | 4 +++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index be017763c..a1b8af316 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -233,13 +233,20 @@ def create_app(application): ) logging.init_app(application) - import hashlib - + # Hopefully will help identify if there is a race condition causing the CSRF errors + # that we have occasionally seen in our environments. for key in ("SECRET_KEY", "DANGEROUS_SALT"): - application.logger.info( - f"{key} Length: {len(application.config[key])}; " - f"SHA-1: {hashlib.sha1(application.config[key].encode('utf8'), usedforsecurity=False).hexdigest()}" - ) + try: + value = application.config[key] + except KeyError: + application.logger.error(f"Env Var {key} doesn't exist.") + else: + try: + data_len = len(value.strip()) + except TypeError: + application.logger.error(f"Env Var {key} invalid type: {type(value)}") + else: + application.logger.info(f"Env Var {key} Length: {data_len}") login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default" diff --git a/app/config.py b/app/config.py index fc358549d..fc9248e05 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = False # If enabled, pytest breaks. ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -164,6 +164,8 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True + + # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. WTF_CSRF_ENABLED = True # buckets - mirror staging From 1a385c6346db4c721a7ba75b710d5eb7876df4cd Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 10:58:48 -0400 Subject: [PATCH 05/11] More cleanup stuff. Signed-off-by: Cliff Hill --- .ds.baseline | 2 +- app/config.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 82ab59b8d..d91a2c576 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 111, + "line_number": 112, "is_secret": false } ], diff --git a/app/config.py b/app/config.py index fc9248e05..97c3ab9ba 100644 --- a/app/config.py +++ b/app/config.py @@ -1,3 +1,4 @@ +from concurrent.futures import ThreadPoolExecutor import json from os import getenv @@ -146,6 +147,9 @@ class Production(Config): class Staging(Production): HEADER_COLOUR = "#00ff00" # $green + # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. + WTF_CSRF_ENABLED = True + class E2ETest(Staging): """ @@ -165,8 +169,8 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. - WTF_CSRF_ENABLED = True + # Disabling CSRF for e2e because things break (a11y & dynamic scan) + WTF_CSRF_ENABLED = False # buckets - mirror staging CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( From fbc51a2c0cd815ebe35b116b38a64440f96f957e Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:00:36 -0400 Subject: [PATCH 06/11] Changest made to fix things. Signed-off-by: Cliff Hill --- .ds.baseline | 2 +- app/config.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index d91a2c576..82ab59b8d 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 112, + "line_number": 111, "is_secret": false } ], diff --git a/app/config.py b/app/config.py index 97c3ab9ba..095561755 100644 --- a/app/config.py +++ b/app/config.py @@ -1,4 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor import json from os import getenv @@ -119,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False # If enabled, pytest breaks. + WTF_CSRF_ENABLED = False # If enabled, pytest breaks. ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" From cdf4a4059169cb07b591b7b37f0fcb46a3a6fec2 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:12:59 -0400 Subject: [PATCH 07/11] Stuff was done. Signed-off-by: Cliff Hill --- app/config.py | 7 +------ poetry.lock | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/config.py b/app/config.py index 095561755..77138ca16 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False # If enabled, pytest breaks. + WTF_CSRF_ENABLED = False ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -146,9 +146,6 @@ class Production(Config): class Staging(Production): HEADER_COLOUR = "#00ff00" # $green - # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. - WTF_CSRF_ENABLED = True - class E2ETest(Staging): """ @@ -167,8 +164,6 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - - # Disabling CSRF for e2e because things break (a11y & dynamic scan) WTF_CSRF_ENABLED = False # buckets - mirror staging diff --git a/poetry.lock b/poetry.lock index d73c31d04..e4cec63a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1344,9 +1344,13 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2527,6 +2531,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From f13898442c55a5d755b3cb1d752bd0da6a05fd63 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:37:02 -0400 Subject: [PATCH 08/11] Reversing poetry.lock changes. Signed-off-by: Cliff Hill --- poetry.lock | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index e4cec63a8..f77b03707 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1344,13 +1344,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -1685,6 +1681,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2531,7 +2528,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From 79bf8d12253e0dadfc5e12ad942db6d1ec04bdcf Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 12:08:02 -0400 Subject: [PATCH 09/11] Better exception handling. Signed-off-by: Cliff Hill --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index a1b8af316..b2d006fe7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -243,7 +243,7 @@ def create_app(application): else: try: data_len = len(value.strip()) - except TypeError: + except {TypeError, AttributeError}: application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: application.logger.info(f"Env Var {key} Length: {data_len}") From 708af19697e5d6f462c34209e6a8fffd16af67b8 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 12:15:00 -0400 Subject: [PATCH 10/11] Fixes. Signed-off-by: Cliff Hill --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index b2d006fe7..964f5c954 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -243,7 +243,7 @@ def create_app(application): else: try: data_len = len(value.strip()) - except {TypeError, AttributeError}: + except (TypeError, AttributeError): application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: application.logger.info(f"Env Var {key} Length: {data_len}") From 56abd154c1ec255addf6a70803cb7f825487688e Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 18 Jun 2024 10:07:00 -0400 Subject: [PATCH 11/11] Now not revealing env var lengths. Signed-off-by: Cliff Hill --- app/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 964f5c954..76f3664cd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -246,7 +246,10 @@ def create_app(application): except (TypeError, AttributeError): application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: - application.logger.info(f"Env Var {key} Length: {data_len}") + if data_len: + application.logger.info(f"Env Var {key} is a non-zero length.") + else: + application.logger.error(f"Env Var {key} is empty.") login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default"