From 06aba23adbb281ff64a82059a07ffde95c361e6d Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 12 Apr 2022 14:40:09 +0100 Subject: [PATCH 1/4] Remove redundant postgres CloudFoundry fixture --- tests/app/test_cloudfoundry_config.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 6a8a128e4..d77887502 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -10,20 +10,13 @@ from app.cloudfoundry_config import ( @pytest.fixture -def postgres_config(): - return [ - { +def cloudfoundry_config(): + return { + 'postgres': [{ 'credentials': { 'uri': 'postgres uri' } - } - ] - - -@pytest.fixture -def cloudfoundry_config(postgres_config): - return { - 'postgres': postgres_config, + }], 'user-provided': [] } From 1f83113e748963cda9688d88a5d36dd7f9a54c1f Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 12 Apr 2022 14:46:47 +0100 Subject: [PATCH 2/4] Move setting VCAP_SERVICES out of fixture This was inconsistent with the source data for the fixture being overidden in some of the tests. We only need to set it in the env once, so it makes sense to just put the code there. --- tests/app/test_cloudfoundry_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index d77887502..df72fdad3 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -22,12 +22,12 @@ def cloudfoundry_config(): @pytest.fixture -def cloudfoundry_environ(os_environ, cloudfoundry_config): - os.environ['VCAP_SERVICES'] = json.dumps(cloudfoundry_config) +def vcap_application(os_environ): os.environ['VCAP_APPLICATION'] = '{"space_name": "🚀🌌"}' -def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_environ): +def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_config, vcap_application): + os.environ['VCAP_SERVICES'] = json.dumps(cloudfoundry_config) extract_cloudfoundry_config() assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri' @@ -35,7 +35,7 @@ def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_environ): assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log' -def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config, cloudfoundry_environ): +def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config, vcap_application): cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}} cloudfoundry_config['user-provided'].append({ 'name': 'bar', 'credentials': {'bar': 'bar'} From fb405977fac2fd76be21c2b0afa334a1209c2ea4 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 12 Apr 2022 14:48:08 +0100 Subject: [PATCH 3/4] Allow REDIS_URL to optionally come from PaaS This is to support a migration from Redislabs to PaaS native Redis, allowing us to toggle between old and new using the env vars for the instance - without needing to change the code. --- app/cloudfoundry_config.py | 3 +++ tests/app/test_cloudfoundry_config.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 84af68c74..d1418e1a0 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -16,6 +16,9 @@ def set_config_env_vars(vcap_services): # Postgres config os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'].replace('postgres', 'postgresql') + # Redis config + if 'redis' in vcap_services: + os.environ['REDIS_URL'] = vcap_services['redis'][0]['credentials']['uri'] vcap_application = json.loads(os.environ['VCAP_APPLICATION']) os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index df72fdad3..9013bd0bf 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -17,6 +17,11 @@ def cloudfoundry_config(): 'uri': 'postgres uri' } }], + 'redis': [{ + 'credentials': { + 'uri': 'redis uri' + } + }], 'user-provided': [] } @@ -31,6 +36,7 @@ def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_config, v extract_cloudfoundry_config() assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri' + assert os.environ['REDIS_URL'] == 'redis uri' assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌' assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log' @@ -45,3 +51,9 @@ def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config, vcap_a assert 'foo' not in os.environ assert 'bar' not in os.environ + + +def test_set_config_env_vars_copes_if_redis_not_set(cloudfoundry_config, vcap_application): + del cloudfoundry_config['redis'] + set_config_env_vars(cloudfoundry_config) + assert 'REDIS_URL' not in os.environ From a2cbe2032565c61b971659aed28ebd4b096ea87d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 23 Feb 2022 13:06:12 +0000 Subject: [PATCH 4/4] fix rediss ssl eventlet sslerror bug eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type :scream: Note: This code should _only_ be called when we're using eventlets, or we'll run into issues with regular code failing with max recursion errors. With that in mind we put this code in gunicorn_config, as that isn't imported when we run celery or run flask locally. --- gunicorn_config.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gunicorn_config.py b/gunicorn_config.py index 62223af54..c4844df4b 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -2,6 +2,8 @@ import os import sys import traceback import gunicorn +import eventlet +import socket from gds_metrics.gunicorn import child_exit # noqa @@ -30,3 +32,22 @@ def on_exit(server): def worker_int(worker): worker.log.info("worker: received SIGINT {}".format(worker.pid)) + + +def fix_ssl_monkeypatching(): + """ + eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently + a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However + eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not + ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type + :scream: + + https://github.com/eventlet/eventlet/issues/692 + """ + # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again. + # eventlet.monkey_patch can be called multiple times without issue + eventlet.monkey_patch() + eventlet.green.ssl.timeout_exc = socket.timeout + + +fix_ssl_monkeypatching()