From d9abca9dbeaf5bb532a9d081a1bc959ba8b38280 Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 15 Aug 2023 01:56:48 -0400 Subject: [PATCH 1/5] Fix comments and imports causing build errors Signed-off-by: Carlo Costino --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e59b45f18..88b849987 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,6 @@ from alembic.config import Config from flask import Flask from app import create_app -from app.config import configs from app.dao.provider_details_dao import get_provider_details_by_identifier @@ -95,9 +94,10 @@ def _notify_db(notify_api): `notify_db_session` fixture which also cleans up any data you've got left over after your test run. """ - # create a database for this worker thread - + # Create a database for this worker thread; note that the + # SQLALCHEMY_DATABASE_URI config variable was already reset with the + # worker ID in the notify_app fixture method. from flask import current_app - #current_app.config['SQLALCHEMY_DATABASE_URI'] += '_{}'.format(worker_id) create_test_db(current_app.config['SQLALCHEMY_DATABASE_URI']) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) From 5519d17e09747e098c0111e086e8b4d0b8a0054b Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 15 Aug 2023 09:58:54 -0400 Subject: [PATCH 2/5] Fix database URI for concurrent test workers Signed-off-by: Carlo Costino --- tests/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 88b849987..e2ffb0bd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -88,16 +88,18 @@ def create_test_db(database_uri): @pytest.fixture(scope='session') -def _notify_db(notify_api): +def _notify_db(notify_api, worker_id): """ Manages the connection to the database. Generally this shouldn't be used, instead you should use the `notify_db_session` fixture which also cleans up any data you've got left over after your test run. """ - # Create a database for this worker thread; note that the - # SQLALCHEMY_DATABASE_URI config variable was already reset with the - # worker ID in the notify_app fixture method. + # Create a database for this worker thread; note that we still have + # to reset it here to point to the correct database. from flask import current_app + current_app.config['SQLALCHEMY_DATABASE_URI'] += '_{}'.format( + worker_id + ) create_test_db(current_app.config['SQLALCHEMY_DATABASE_URI']) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) From 17e13c3b3b547cfbf5d85d2e6d160e3eb3eef60e Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 15 Aug 2023 10:58:16 -0400 Subject: [PATCH 3/5] Round 2 of attempting to fix CI test runs Signed-off-by: Carlo Costino --- tests/conftest.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e2ffb0bd2..37b88a62f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,23 +106,22 @@ def _notify_db(notify_api, worker_id): ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations') config = Config(ALEMBIC_CONFIG + '/alembic.ini') config.set_main_option("script_location", ALEMBIC_CONFIG) + config.set_main_option( + "sqlalchemy.url", + current_app.config['SQLALCHEMY_DATABASE_URI'] + ) - with notify_api.app_context(): + with notify_api.app_context() as app_context: + # Run DB migrations. upgrade(config, 'head') - # Retrieve the DB object from the initialized app. - db = notify_api.extensions['sqlalchemy'] + # Modify the database connection URL to point to the correct + # test database. + db = app_context.app.extensions['sqlalchemy'] + db.engine.url = current_app.config['SQLALCHEMY_DATABASE_URI'] - # Check the DB name. - assert 'test_notification_api' in db.engine.url.database, 'dont run tests against main db' - - # Modify the URL to point to the correct test database. - db.engine.url = current_app.config['SQLALCHEMY_DATABASE_URI'] - - yield db - - db.session.remove() - db.engine.dispose() + # Return the DB object to the test calling for it. + yield db @pytest.fixture(scope='function') From 206d5fccb76a7e450d968898878f9c3ebd6d4f2b Mon Sep 17 00:00:00 2001 From: Carlo Costino Date: Tue, 15 Aug 2023 18:03:49 -0400 Subject: [PATCH 4/5] Simplify test config for now This is in prep for switching to running a single test worker for the time being. We will spend time figuring out how to get multiple concurrent workers going again separately as we seem to have several issues taking place right now with that type of test setup. Signed-off-by: Carlo Costino --- tests/conftest.py | 88 ++++++++++------------------------------------- 1 file changed, 18 insertions(+), 70 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 37b88a62f..ad3ceebec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,6 @@ import os from contextlib import contextmanager import pytest -import sqlalchemy from alembic.command import upgrade from alembic.config import Config from flask import Flask @@ -12,35 +11,14 @@ from app.dao.provider_details_dao import get_provider_details_by_identifier @pytest.fixture(scope='session') -def notify_app(worker_id): +def notify_app(): app = Flask('test') - - # Override the SQLALCHEMY_DATABASE_URI config before the app is - # initialized to account for Flask-SQLAlchemy 3.0.x changes. - # What is ultimately happening is the create_engine call made with - # SQLAlchemy itself is now only happening at the time of calling - # init_app with Flask instead of at the time it is first accessed, - # which the _notify_db fixture method was relying on. - - # See the following for more information: - # https://github.com/pallets-eco/flask-sqlalchemy/pull/1087 - # https://flask-sqlalchemy.palletsprojects.com/en/3.0.x/api/#module-flask_sqlalchemy - app.config['SQLALCHEMY_DATABASE_URI'] = '{}_{}'.format( - os.getenv('SQLALCHEMY_DATABASE_TEST_URI', '').replace('postgres://', 'postgresql://'), - worker_id - ) - create_app(app) return app @pytest.fixture(scope='session') def notify_api(notify_app): - # deattach server-error error handlers - error_handler_spec looks like: - # {'blueprint_name': { - # status_code: [error_handlers], - # None: { ExceptionClass: error_handler } - # }} for error_handlers in notify_app.error_handler_spec.values(): error_handlers.pop(500, None) if None in error_handlers: @@ -66,63 +44,33 @@ def client(notify_api): yield client -def create_test_db(database_uri): - # get the - db_uri_parts = database_uri.split('/') - postgres_db_uri = '/'.join(db_uri_parts[:-1] + ['postgres']) - - postgres_db = sqlalchemy.create_engine( - postgres_db_uri, - echo=False, - isolation_level='AUTOCOMMIT', - client_encoding='utf8' - ) - try: - result = postgres_db.execute(sqlalchemy.sql.text('CREATE DATABASE {}'.format(db_uri_parts[-1]))) - result.close() - except sqlalchemy.exc.ProgrammingError: - # database "test_notification_api_master" already exists - pass - finally: - postgres_db.dispose() - - @pytest.fixture(scope='session') -def _notify_db(notify_api, worker_id): +def _notify_db(notify_api): """ Manages the connection to the database. Generally this shouldn't be used, instead you should use the `notify_db_session` fixture which also cleans up any data you've got left over after your test run. """ - - # Create a database for this worker thread; note that we still have - # to reset it here to point to the correct database. - from flask import current_app - current_app.config['SQLALCHEMY_DATABASE_URI'] += '_{}'.format( - worker_id - ) - create_test_db(current_app.config['SQLALCHEMY_DATABASE_URI']) - - BASE_DIR = os.path.dirname(os.path.dirname(__file__)) - ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations') - config = Config(ALEMBIC_CONFIG + '/alembic.ini') - config.set_main_option("script_location", ALEMBIC_CONFIG) - config.set_main_option( - "sqlalchemy.url", - current_app.config['SQLALCHEMY_DATABASE_URI'] - ) - with notify_api.app_context() as app_context: - # Run DB migrations. + db = app_context.app.extensions['sqlalchemy'] + assert 'test_notification_api' in db.engine.url.database, 'dont run tests against main db' + + BASE_DIR = os.path.dirname(os.path.dirname(__file__)) + ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations') + config = Config(ALEMBIC_CONFIG + '/alembic.ini') + config.set_main_option('script_location', ALEMBIC_CONFIG) + config.set_main_option( + 'sqlalchemy.url', + app_context.app.config['SQLALCHEMY_DATABASE_URI'] + ) + + # Run migrations on the test database. upgrade(config, 'head') - # Modify the database connection URL to point to the correct - # test database. - db = app_context.app.extensions['sqlalchemy'] - db.engine.url = current_app.config['SQLALCHEMY_DATABASE_URI'] - - # Return the DB object to the test calling for it. yield db + db.session.remove() + db.engine.dispose() + @pytest.fixture(scope='function') def sms_providers(_notify_db): From 619d5fe8c5610faf02d0be569f182309d9d3bb79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:01:00 +0000 Subject: [PATCH 5/5] Bump moto from 3.1.9 to 4.1.14 Bumps [moto](https://github.com/getmoto/moto) from 3.1.9 to 4.1.14. - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/3.1.9...4.1.14) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Pipfile | 2 +- Pipfile.lock | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Pipfile b/Pipfile index 0e391c237..b870cb6d2 100644 --- a/Pipfile +++ b/Pipfile @@ -69,7 +69,7 @@ exceptiongroup = "==1.1.2" flake8 = "==4.0.1" flake8-bugbear = "==23.3.12" isort = "==5.10.1" -moto = "==3.1.9" +moto = "==4.1.14" pytest = "==7.4.0" pytest-env = "==0.6.2" pytest-mock = "==3.11.1" diff --git a/Pipfile.lock b/Pipfile.lock index c9e1141b6..fec41af87 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0b4231d3027251f5b90eced0dd6099091ac6536237ac3baa8e905a07af253194" + "sha256": "5e5181f50358347d69bdf2e24b58e7a8bf84c76f1b32196cf67a43b8c23367d6" }, "pipfile-spec": 6, "requires": { @@ -894,7 +894,7 @@ "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac", "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==3.0.39" }, "psycopg2-binary": { @@ -1139,7 +1139,7 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], - "markers": "python_version >= '3.5' and python_version < '4'", + "markers": "python_version < '4.0' and python_full_version >= '3.5.0'", "version": "==4.7.2" }, "s3transfer": { @@ -2062,11 +2062,11 @@ }, "moto": { "hashes": [ - "sha256:8928ec168e5fd88b1127413b2fa570a80d45f25182cdad793edd208d07825269", - "sha256:ba683e70950b6579189bc12d74c1477aa036c090c6ad8b151a22f5896c005113" + "sha256:545afeb4df94dfa730e2d7e87366dc26b4a33c2891f462cbb049f040c80ed1ec", + "sha256:7d3bd748a34641715ba469c761f72fb8ec18f349987c98f5a0f9be85a07a9911" ], "index": "pypi", - "version": "==3.1.9" + "version": "==4.1.14" }, "msgpack": { "hashes": [ @@ -2276,7 +2276,7 @@ "sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526", "sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==32.0.1" }, "pluggy": { @@ -2407,13 +2407,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, - "pytz": { - "hashes": [ - "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", - "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" - ], - "version": "==2023.3" - }, "pyyaml": { "hashes": [ "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", @@ -2489,7 +2482,7 @@ "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808", "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==13.5.2" }, "s3transfer": {