From f539cad4b4a831a7c1a75c6208cfb5da72c91dd5 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 11:55:26 -0400 Subject: [PATCH 01/19] Added some logging up front when the app is initialized to see if DANGEROUS_SALT and SECRET_KEY are both available and have data in them. Signed-off-by: Cliff Hill --- app/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index e99a5ae51..be017763c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -233,6 +233,14 @@ def create_app(application): ) logging.init_app(application) + import hashlib + + for key in ("SECRET_KEY", "DANGEROUS_SALT"): + application.logger.info( + f"{key} Length: {len(application.config[key])}; " + f"SHA-1: {hashlib.sha1(application.config[key].encode('utf8'), usedforsecurity=False).hexdigest()}" + ) + login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default" login_manager.session_protection = None From ec9a36e0dd01819dd6e28e86302f256b352c0535 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 15:09:28 -0400 Subject: [PATCH 02/19] Activating CSRF for staging/demo/dev environments. Signed-off-by: Cliff Hill --- app/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config.py b/app/config.py index 77138ca16..8922506f1 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = True ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -164,7 +164,7 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = True # buckets - mirror staging CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( From 9e94b5a242cc3971c4cb1af1f0e6318008fbd3c0 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 7 Jun 2024 16:10:18 -0400 Subject: [PATCH 03/19] saabling CSRF in dev because it breaks all the tests. Signed-off-by: Cliff Hill --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 8922506f1..fc358549d 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = True + WTF_CSRF_ENABLED = False ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" From 530206e33a1dd19941e5405b84e1edc7e7c16ed5 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 10:13:15 -0400 Subject: [PATCH 04/19] Cleaning up the logging, gave some additional notes, removed hashing. Signed-off-by: Cliff Hill --- app/__init__.py | 19 +++++++++++++------ app/config.py | 4 +++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index be017763c..a1b8af316 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -233,13 +233,20 @@ def create_app(application): ) logging.init_app(application) - import hashlib - + # Hopefully will help identify if there is a race condition causing the CSRF errors + # that we have occasionally seen in our environments. for key in ("SECRET_KEY", "DANGEROUS_SALT"): - application.logger.info( - f"{key} Length: {len(application.config[key])}; " - f"SHA-1: {hashlib.sha1(application.config[key].encode('utf8'), usedforsecurity=False).hexdigest()}" - ) + try: + value = application.config[key] + except KeyError: + application.logger.error(f"Env Var {key} doesn't exist.") + else: + try: + data_len = len(value.strip()) + except TypeError: + application.logger.error(f"Env Var {key} invalid type: {type(value)}") + else: + application.logger.info(f"Env Var {key} Length: {data_len}") login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default" diff --git a/app/config.py b/app/config.py index fc358549d..fc9248e05 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False + WTF_CSRF_ENABLED = False # If enabled, pytest breaks. ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -164,6 +164,8 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True + + # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. WTF_CSRF_ENABLED = True # buckets - mirror staging From 1a385c6346db4c721a7ba75b710d5eb7876df4cd Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 10:58:48 -0400 Subject: [PATCH 05/19] More cleanup stuff. Signed-off-by: Cliff Hill --- .ds.baseline | 2 +- app/config.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 82ab59b8d..d91a2c576 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 111, + "line_number": 112, "is_secret": false } ], diff --git a/app/config.py b/app/config.py index fc9248e05..97c3ab9ba 100644 --- a/app/config.py +++ b/app/config.py @@ -1,3 +1,4 @@ +from concurrent.futures import ThreadPoolExecutor import json from os import getenv @@ -146,6 +147,9 @@ class Production(Config): class Staging(Production): HEADER_COLOUR = "#00ff00" # $green + # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. + WTF_CSRF_ENABLED = True + class E2ETest(Staging): """ @@ -165,8 +169,8 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. - WTF_CSRF_ENABLED = True + # Disabling CSRF for e2e because things break (a11y & dynamic scan) + WTF_CSRF_ENABLED = False # buckets - mirror staging CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( From fbc51a2c0cd815ebe35b116b38a64440f96f957e Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:00:36 -0400 Subject: [PATCH 06/19] Changest made to fix things. Signed-off-by: Cliff Hill --- .ds.baseline | 2 +- app/config.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index d91a2c576..82ab59b8d 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 112, + "line_number": 111, "is_secret": false } ], diff --git a/app/config.py b/app/config.py index 97c3ab9ba..095561755 100644 --- a/app/config.py +++ b/app/config.py @@ -1,4 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor import json from os import getenv @@ -119,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False # If enabled, pytest breaks. + WTF_CSRF_ENABLED = False # If enabled, pytest breaks. ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" From cdf4a4059169cb07b591b7b37f0fcb46a3a6fec2 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:12:59 -0400 Subject: [PATCH 07/19] Stuff was done. Signed-off-by: Cliff Hill --- app/config.py | 7 +------ poetry.lock | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/config.py b/app/config.py index 095561755..77138ca16 100644 --- a/app/config.py +++ b/app/config.py @@ -118,7 +118,7 @@ class Development(Config): class Test(Development): TESTING = True - WTF_CSRF_ENABLED = False # If enabled, pytest breaks. + WTF_CSRF_ENABLED = False ASSET_DOMAIN = "static.example.com" ASSET_PATH = "https://static.example.com/" @@ -146,9 +146,6 @@ class Production(Config): class Staging(Production): HEADER_COLOUR = "#00ff00" # $green - # Attempting to enable CSRF in staging with the hopes that we can duplicate errors. - WTF_CSRF_ENABLED = True - class E2ETest(Staging): """ @@ -167,8 +164,6 @@ class E2ETest(Staging): # Borrowed from test environment TESTING = True - - # Disabling CSRF for e2e because things break (a11y & dynamic scan) WTF_CSRF_ENABLED = False # buckets - mirror staging diff --git a/poetry.lock b/poetry.lock index d73c31d04..e4cec63a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1344,9 +1344,13 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, + {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, + {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -2527,6 +2531,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From f13898442c55a5d755b3cb1d752bd0da6a05fd63 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 11:37:02 -0400 Subject: [PATCH 08/19] Reversing poetry.lock changes. Signed-off-by: Cliff Hill --- poetry.lock | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index e4cec63a8..f77b03707 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1344,13 +1344,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -1685,6 +1681,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2531,7 +2528,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From 79bf8d12253e0dadfc5e12ad942db6d1ec04bdcf Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 12:08:02 -0400 Subject: [PATCH 09/19] Better exception handling. Signed-off-by: Cliff Hill --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index a1b8af316..b2d006fe7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -243,7 +243,7 @@ def create_app(application): else: try: data_len = len(value.strip()) - except TypeError: + except {TypeError, AttributeError}: application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: application.logger.info(f"Env Var {key} Length: {data_len}") From 708af19697e5d6f462c34209e6a8fffd16af67b8 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 17 Jun 2024 12:15:00 -0400 Subject: [PATCH 10/19] Fixes. Signed-off-by: Cliff Hill --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index b2d006fe7..964f5c954 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -243,7 +243,7 @@ def create_app(application): else: try: data_len = len(value.strip()) - except {TypeError, AttributeError}: + except (TypeError, AttributeError): application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: application.logger.info(f"Env Var {key} Length: {data_len}") From 56abd154c1ec255addf6a70803cb7f825487688e Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 18 Jun 2024 10:07:00 -0400 Subject: [PATCH 11/19] Now not revealing env var lengths. Signed-off-by: Cliff Hill --- app/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 964f5c954..76f3664cd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -246,7 +246,10 @@ def create_app(application): except (TypeError, AttributeError): application.logger.error(f"Env Var {key} invalid type: {type(value)}") else: - application.logger.info(f"Env Var {key} Length: {data_len}") + if data_len: + application.logger.info(f"Env Var {key} is a non-zero length.") + else: + application.logger.error(f"Env Var {key} is empty.") login_manager.login_view = "main.sign_in" login_manager.login_message_category = "default" From 5429af16d48e16765d3dbb8e6efc63c4b23d9c7c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 10:45:22 -0700 Subject: [PATCH 12/19] pin python to 3.12.2 --- .github/actions/setup-project/action.yml | 4 ++-- poetry.lock | 5 +++-- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml index b2d582986..859254091 100644 --- a/.github/actions/setup-project/action.yml +++ b/.github/actions/setup-project/action.yml @@ -9,10 +9,10 @@ runs: sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ libcurl4-openssl-dev - - name: Set up Python 3.12 + - name: Set up Python 3.12.2 uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: "3.12.2" - name: Install poetry shell: bash run: pip install poetry diff --git a/poetry.lock b/poetry.lock index d73c31d04..b48a5c53e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1681,6 +1681,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3110,5 +3111,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "^3.12.2" -content-hash = "b45f2c38493f81bd7fc9d4bfd294b001d71e4082380eb0851d4f3ea8dcdb949c" +python-versions = "==3.12.2" +content-hash = "13525f32422aad8324a3bcb3629d326cc847029800d5518e1efd047b887afa8b" diff --git a/pyproject.toml b/pyproject.toml index 4dd46365a..fbeb26903 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" package-mode = false [tool.poetry.dependencies] -python = "^3.12.2" +python = "==3.12.2" ago = "~=0.0.95" beautifulsoup4 = "^4.12.3" blinker = "~=1.8" From 27befe08067338da51b3cdae0789d795e2f1af46 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Thu, 20 Jun 2024 14:32:19 -0400 Subject: [PATCH 13/19] Fix runtime.txt for cloud.gov and Python This changeset updates the runtime.txt file to match the recent Python version pinning that we had to do to fix a dependency issue. Signed-off-by: Carlo Costino --- runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime.txt b/runtime.txt index 64f28603a..6e797d468 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.12.x +python-3.12.2 From 7b5fdd41df78a45046631dd19a68a190139888aa Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 11:50:51 -0700 Subject: [PATCH 14/19] fix raw set --- app/main/views/register.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/main/views/register.py b/app/main/views/register.py index a7e188f0c..761582b3d 100644 --- a/app/main/views/register.py +++ b/app/main/views/register.py @@ -130,10 +130,10 @@ def put_invite_data_in_redis( ): ttl = 60 * 15 # 15 minutes - redis_client.raw_set(f"invitedata-{state}", json.dumps(invite_data), ex=ttl) - redis_client.raw_set(f"user_email-{state}", user_email, ex=ttl) - redis_client.raw_set(f"user_uuid-{state}", user_uuid, ex=ttl) - redis_client.raw_set( + redis_client.set(f"invitedata-{state}", json.dumps(invite_data), ex=ttl) + redis_client.set(f"user_email-{state}", user_email, ex=ttl) + redis_client.set(f"user_uuid-{state}", user_uuid, ex=ttl) + redis_client.set( f"invited_user_email_address-{state}", invited_user_email_address, ex=ttl, From 87c3efd312e720586a65e6c1b3c320a1c56eee6c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 12:37:11 -0700 Subject: [PATCH 15/19] switch to 3.12.3 --- .github/actions/setup-project/action.yml | 4 ++-- poetry.lock | 4 ++-- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml index 859254091..db1540fad 100644 --- a/.github/actions/setup-project/action.yml +++ b/.github/actions/setup-project/action.yml @@ -9,10 +9,10 @@ runs: sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ libcurl4-openssl-dev - - name: Set up Python 3.12.2 + - name: Set up Python 3.12.3 uses: actions/setup-python@v4 with: - python-version: "3.12.2" + python-version: "3.12.3" - name: Install poetry shell: bash run: pip install poetry diff --git a/poetry.lock b/poetry.lock index b48a5c53e..f77b03707 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3111,5 +3111,5 @@ files = [ [metadata] lock-version = "2.0" -python-versions = "==3.12.2" -content-hash = "13525f32422aad8324a3bcb3629d326cc847029800d5518e1efd047b887afa8b" +python-versions = "^3.12.2" +content-hash = "b45f2c38493f81bd7fc9d4bfd294b001d71e4082380eb0851d4f3ea8dcdb949c" diff --git a/pyproject.toml b/pyproject.toml index fbeb26903..4dd46365a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" package-mode = false [tool.poetry.dependencies] -python = "==3.12.2" +python = "^3.12.2" ago = "~=0.0.95" beautifulsoup4 = "^4.12.3" blinker = "~=1.8" From 3b6ccc031ab42d4ff6355c80ef3e9ca752c5a25a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 20 Jun 2024 13:16:45 -0700 Subject: [PATCH 16/19] fix format? --- tests/app/main/views/test_index.py | 9 ++++++--- tests/end_to_end/test_landing_and_sign_in_pages.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/app/main/views/test_index.py b/tests/app/main/views/test_index.py index 1e08cd606..154106250 100644 --- a/tests/app/main/views/test_index.py +++ b/tests/app/main/views/test_index.py @@ -19,9 +19,12 @@ def test_non_logged_in_user_can_see_homepage( "Reach people where they are with government-powered text messages" ) - assert page.select_one( - "a.usa-button.login-button.login-button--primary.margin-right-2" - ).text == "Sign in with \n" + assert ( + page.select_one( + "a.usa-button.login-button.login-button--primary.margin-right-2" + ).text + == "Sign in with \n" + ) assert page.select_one("meta[name=description]") is not None # This area is hidden for the pilot # assert normalize_spaces(page.select_one('#whos-using-notify').text) == ( diff --git a/tests/end_to_end/test_landing_and_sign_in_pages.py b/tests/end_to_end/test_landing_and_sign_in_pages.py index 513269f51..4dcab63a1 100644 --- a/tests/end_to_end/test_landing_and_sign_in_pages.py +++ b/tests/end_to_end/test_landing_and_sign_in_pages.py @@ -31,7 +31,7 @@ def test_landing_page(end_to_end_context): expect(benefits_studio_email).to_be_visible() # Check to make sure the sign-in button and email links are correct. - href_value = sign_in_button.get_attribute('href') + href_value = sign_in_button.get_attribute("href") assert href_value is not None, "The sign-in button does not have an href attribute" expect(benefits_studio_email).to_have_attribute( "href", "mailto:tts-benefits-studio@gsa.gov" From 31cd55725fc26ab3cd27434282a30a17f2bf271d Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Thu, 20 Jun 2024 18:28:26 -0400 Subject: [PATCH 17/19] Update runtime.txt to reference Python 3.12.3 Signed-off-by: Carlo Costino --- runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime.txt b/runtime.txt index 6e797d468..4ddc7cd66 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.12.2 +python-3.12.3 From 81dd764a54d4c493dda18c37f6ddcd4e57d4d213 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 21 Jun 2024 09:22:20 -0700 Subject: [PATCH 18/19] fix gunicorn config --- gunicorn_config.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 4f928a467..f9eb4f17d 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -5,7 +5,10 @@ import multiprocessing import gunicorn # Let gunicorn figure out the right number of workers -workers = multiprocessing.cpu_count() * 2 + 1 +# The recommended formula is cpu_count() * 2 + 1 +# but we have an unusual configuration with a lot of cpus and not much memory +# so adjust it. +workers = multiprocessing.cpu_count() worker_class = "eventlet" bind = "0.0.0.0:{}".format(os.getenv("PORT")) disable_redirect_access_to_syslog = True @@ -16,23 +19,3 @@ def worker_abort(worker): worker.log.info("worker received ABORT") for stack in sys._current_frames().values(): worker.log.error("".join(traceback.format_stack(stack))) - - -# This issue is fixed in the 22.0.0 release, which we are using -# See github issue for details -# def fix_ssl_monkeypatching(): -# """ -# eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently -# a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However -# eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not -# ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type -# :scream: -# https://github.com/eventlet/eventlet/issues/692 -# """ -# # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again. -# # eventlet.monkey_patch can be called multiple times without issue -# eventlet.monkey_patch() -# eventlet.green.ssl.timeout_exc = socket.timeout - - -# fix_ssl_monkeypatching() From a887f3f5273ab97c7fe893a708a6bb0824e5d181 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Mon, 24 Jun 2024 12:07:28 -0400 Subject: [PATCH 19/19] Increased production memory to 2 GB This changeset increases our memory for the API app and its corresponding workers to 2 GB each. Signed-off-by: Carlo Costino --- deploy-config/production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 18b5cfdcc..b38094f28 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -1,6 +1,6 @@ env: production instances: 2 -memory: 1G +memory: 2G public_admin_route: beta.notify.gov cloud_dot_gov_route: notify.app.cloud.gov redis_enabled: 1