From 5b405d41aa68776e4b85ef82defdb09d5e06a3c7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 12:07:42 -0700 Subject: [PATCH 001/128] break test so we can see coverage numbers --- tests/app/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 46dd2b0c1..8375aa7a9 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -437,7 +437,8 @@ def test_download_csv_file_by_name(notify_api, mocker): "NonExistentName", ], ) - mock_download.assert_called_once() + mock_download.assert_not_called() + # mock_download.assert_called_once() def test_promote_user_to_platform_admin_no_result_found( From a0c27975a57ff40a31dfdea37165c008d787f1d4 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 12:40:31 -0700 Subject: [PATCH 002/128] break test so we can see coverage numbers --- tests/app/test_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 8375aa7a9..46dd2b0c1 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -437,8 +437,7 @@ def test_download_csv_file_by_name(notify_api, mocker): "NonExistentName", ], ) - mock_download.assert_not_called() - # mock_download.assert_called_once() + mock_download.assert_called_once() def test_promote_user_to_platform_admin_no_result_found( From 5d265135d35fd1afcccca7ac64280f413d27c109 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 13:08:34 -0700 Subject: [PATCH 003/128] write a test --- app/delivery/send_to_providers.py | 24 +++++++++++--------- tests/app/delivery/test_send_to_providers.py | 17 ++++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index 745b46cab..07763823f 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -98,17 +98,7 @@ def send_sms_to_provider(notification): # TODO This is temporary to test the capability of validating phone numbers # The future home of the validation is TBD - if "+" not in recipient: - recipient_lookup = f"+{recipient}" - else: - recipient_lookup = recipient - if recipient_lookup in current_app.config[ - "SIMULATED_SMS_NUMBERS" - ] and os.getenv("NOTIFY_ENVIRONMENT") in ["development", "test"]: - current_app.logger.info(hilite("#validate-phone-number fired")) - aws_pinpoint_client.validate_phone_number("01", recipient) - else: - current_app.logger.info(hilite("#validate-phone-number not fired")) + _experimentally_validate_phone_numbers(recipient) sender_numbers = get_sender_numbers(notification) if notification.reply_to_text not in sender_numbers: @@ -145,6 +135,18 @@ def send_sms_to_provider(notification): return message_id +def _experimentally_validate_phone_numbers(recipient): + if "+" not in recipient: + recipient_lookup = f"+{recipient}" + else: + recipient_lookup = recipient + if recipient_lookup in current_app.config["SIMULATED_SMS_NUMBERS"] and os.getenv( + "NOTIFY_ENVIRONMENT" + ) in ["development", "test"]: + current_app.logger.info(hilite("#validate-phone-number fired")) + aws_pinpoint_client.validate_phone_number("01", recipient) + + def _get_verify_code(notification): key = f"2facode-{notification.id}".replace(" ", "") recipient = redis_store.get(key) diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index fbea9a2f7..4c0c39890 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -3,6 +3,7 @@ import os from contextlib import suppress from urllib import parse +import pytest from cachetools import TTLCache, cached from flask import current_app @@ -19,6 +20,7 @@ from app.dao.email_branding_dao import dao_get_email_branding_by_id from app.dao.notifications_dao import dao_update_notification from app.dao.provider_details_dao import get_provider_details_by_notification_type from app.dao.service_sms_sender_dao import dao_get_sms_senders_by_service_id +from app.delivery.send_to_providers import _experimentally_validate_phone_numbers from app.enums import BrandType, KeyType, NotificationStatus, NotificationType from app.exceptions import NotificationTechnicalFailureException from app.serialised_models import SerialisedService, SerialisedTemplate @@ -306,3 +308,18 @@ def technical_failure(notification): f"Send {notification.notification_type} for notification id {notification.id} " f"to provider is not allowed: service {notification.service_id} is inactive" ) + + +@pytest.mark.parametrize( + ("recipient", "expected_invoke"), + [ + ("15555555555", False), + ], +) +def test_experimentally_validate_phone_numbers(recipient, expected_invoke, mocker): + mock_pinpoint = mocker.patch("app.delivery.send_to_providers.aws_pinpoint_client") + _experimentally_validate_phone_numbers(recipient) + if expected_invoke: + mock_pinpoint.phone_number_validate.assert_called_once_with("foo") + else: + mock_pinpoint.phone_number_validate.assert_not_called() From 6e73e81201c4b01f0d3147f5c454d052a0acfb6b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 13:40:19 -0700 Subject: [PATCH 004/128] ugh fix tests --- tests/app/delivery/test_send_to_providers.py | 1208 ++++++++++++++---- 1 file changed, 946 insertions(+), 262 deletions(-) diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 4c0c39890..20b0f7186 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -1,313 +1,997 @@ import json -import os -from contextlib import suppress -from urllib import parse +from collections import namedtuple +from unittest.mock import ANY import pytest -from cachetools import TTLCache, cached from flask import current_app +from requests import HTTPError -from app import ( - aws_pinpoint_client, - create_uuid, - db, - notification_provider_clients, - redis_store, +import app +from app import aws_sns_client, notification_provider_clients +from app.cloudfoundry_config import cloud_config +from app.dao import notifications_dao +from app.dao.provider_details_dao import get_provider_details_by_identifier +from app.delivery import send_to_providers +from app.delivery.send_to_providers import ( + _experimentally_validate_phone_numbers, + get_html_email_options, + get_logo_url, ) -from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3 -from app.celery.test_key_tasks import send_email_response, send_sms_response -from app.dao.email_branding_dao import dao_get_email_branding_by_id -from app.dao.notifications_dao import dao_update_notification -from app.dao.provider_details_dao import get_provider_details_by_notification_type -from app.dao.service_sms_sender_dao import dao_get_sms_senders_by_service_id -from app.delivery.send_to_providers import _experimentally_validate_phone_numbers from app.enums import BrandType, KeyType, NotificationStatus, NotificationType from app.exceptions import NotificationTechnicalFailureException -from app.serialised_models import SerialisedService, SerialisedTemplate -from app.utils import hilite, utc_now -from notifications_utils.template import ( - HTMLEmailTemplate, - PlainTextEmailTemplate, - SMSMessageTemplate, +from app.models import EmailBranding, Notification +from app.serialised_models import SerialisedService +from app.utils import utc_now +from tests.app.db import ( + create_email_branding, + create_notification, + create_reply_to_email, + create_service, + create_service_sms_sender, + create_service_with_defined_sms_sender, + create_template, ) -def send_sms_to_provider(notification): - """Final step in the message send flow. - - Get data for recipient, template, - notification and send it to sns. - """ - # we no longer store the personalisation in the db, - # need to retrieve from s3 before generating content - # However, we are still sending the initial verify code through personalisation - # so if there is some value there, don't overwrite it - if not notification.personalisation: - personalisation = get_personalisation_from_s3( - notification.service_id, - notification.job_id, - notification.job_row_number, - ) - notification.personalisation = personalisation - - service = SerialisedService.from_id(notification.service_id) - message_id = None - if not service.active: - technical_failure(notification=notification) - return - - if notification.status == NotificationStatus.CREATED: - # We get the provider here (which is only aws sns) - provider = provider_to_use(NotificationType.SMS, notification.international) - if not provider: - technical_failure(notification=notification) - return - - template_model = SerialisedTemplate.from_id_and_service_id( - template_id=notification.template_id, - service_id=service.id, - version=notification.template_version, - ) - - template = SMSMessageTemplate( - template_model.__dict__, - values=notification.personalisation, - prefix=service.name, - show_prefix=service.prefix_sms, - ) - if notification.key_type == KeyType.TEST: - update_notification_to_sending(notification, provider) - send_sms_response(provider.name, str(notification.id)) - - else: - try: - # End DB session here so that we don't have a connection stuck open waiting on the call - # to one of the SMS providers - # We don't want to tie our DB connections being open to the performance of our SMS - # providers as a slow down of our providers can cause us to run out of DB connections - # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before - # closing the session (as otherwise it would be reopened immediately) - - # We start by trying to get the phone number from a job in s3. If we fail, we assume - # the phone number is for the verification code on login, which is not a job. - recipient = None - # It is our 2facode, maybe - recipient = _get_verify_code(notification) - - if recipient is None: - recipient = get_phone_number_from_s3( - notification.service_id, - notification.job_id, - notification.job_row_number, - ) - - # TODO This is temporary to test the capability of validating phone numbers - # The future home of the validation is TBD - if "+" not in recipient: - recipient_lookup = f"+{recipient}" - else: - recipient_lookup = recipient - if recipient_lookup in current_app.config[ - "SIMULATED_SMS_NUMBERS" - ] and os.getenv("NOTIFY_ENVIRONMENT") in ["development", "test"]: - current_app.logger.info(hilite("#validate-phone-number fired")) - aws_pinpoint_client.validate_phone_number("01", recipient) - else: - current_app.logger.info(hilite("#validate-phone-number not fired")) - - sender_numbers = get_sender_numbers(notification) - if notification.reply_to_text not in sender_numbers: - raise ValueError( - f"{notification.reply_to_text} not in {sender_numbers} #notify-admin-1701" - ) - - send_sms_kwargs = { - "to": recipient, - "content": str(template), - "reference": str(notification.id), - "sender": notification.reply_to_text, - "international": notification.international, - } - db.session.close() # no commit needed as no changes to objects have been made above - - message_id = provider.send_sms(**send_sms_kwargs) - current_app.logger.info(f"got message_id {message_id}") - except Exception as e: - n = notification - msg = f"FAILED send to sms, job_id: {n.job_id} row_number {n.job_row_number} message_id {message_id}" - current_app.logger.exception(hilite(msg)) - - notification.billable_units = template.fragment_count - dao_update_notification(notification) - raise e - else: - # Here we map the job_id and row number to the aws message_id - n = notification - msg = f"Send to aws for job_id {n.job_id} row_number {n.job_row_number} message_id {message_id}" - current_app.logger.info(hilite(msg)) - notification.billable_units = template.fragment_count - update_notification_to_sending(notification, provider) - return message_id +def setup_function(_function): + # pytest will run this function before each test. It makes sure the + # state of the cache is not shared between tests. + send_to_providers.provider_cache.clear() -def _get_verify_code(notification): - key = f"2facode-{notification.id}".replace(" ", "") - recipient = redis_store.get(key) - with suppress(AttributeError): - recipient = recipient.decode("utf-8") - return recipient +@pytest.mark.parametrize( + "international_provider_priority", + ( + # Since there’s only one international provider it should always + # be used, no matter what its priority is set to + 0, + 50, + 100, + ), +) +def test_provider_to_use_should_only_return_sns_for_international( + mocker, + notify_db_session, + international_provider_priority, +): + sns = get_provider_details_by_identifier("sns") + sns.priority = international_provider_priority + + ret = send_to_providers.provider_to_use(NotificationType.SMS, international=True) + + assert ret.name == "sns" -def get_sender_numbers(notification): - possible_senders = dao_get_sms_senders_by_service_id(notification.service_id) - sender_numbers = [] - for possible_sender in possible_senders: - sender_numbers.append(possible_sender.sms_sender) - return sender_numbers +def test_provider_to_use_raises_if_no_active_providers( + mocker, restore_provider_details +): + sns = get_provider_details_by_identifier("sns") + sns.active = False + + # flake8 doesn't like raises with a generic exception + try: + send_to_providers.provider_to_use(NotificationType.SMS) + assert 1 == 0 + except Exception: + assert 1 == 1 -def send_email_to_provider(notification): - # Someone needs an email, possibly new registration - recipient = redis_store.get(f"email-address-{notification.id}") - recipient = recipient.decode("utf-8") - personalisation = redis_store.get(f"email-personalisation-{notification.id}") - if personalisation: - personalisation = personalisation.decode("utf-8") - notification.personalisation = json.loads(personalisation) +def test_should_send_personalised_template_to_correct_sms_provider_and_persist( + sample_sms_template_with_html, mocker +): - service = SerialisedService.from_id(notification.service_id) - if not service.active: - technical_failure(notification=notification) - return - if notification.status == NotificationStatus.CREATED: - provider = provider_to_use(NotificationType.EMAIL, False) - template_dict = SerialisedTemplate.from_id_and_service_id( - template_id=notification.template_id, - service_id=service.id, - version=notification.template_version, - ).__dict__ + mocker.patch("app.delivery.send_to_providers._get_verify_code", return_value=None) + db_notification = create_notification( + template=sample_sms_template_with_html, + personalisation={}, + status=NotificationStatus.CREATED, + reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(), + ) - html_email = HTMLEmailTemplate( - template_dict, - values=notification.personalisation, - **get_html_email_options(service), - ) + mocker.patch("app.aws_sns_client.send_sms") - plain_text_email = PlainTextEmailTemplate( - template_dict, values=notification.personalisation - ) + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "2028675309" - if notification.key_type == KeyType.TEST: - notification.reference = str(create_uuid()) - update_notification_to_sending(notification, provider) - send_email_response(notification.reference, recipient) - else: - from_address = ( - f'"{service.name}" <{service.email_from}@' - f'{current_app.config["NOTIFY_EMAIL_DOMAIN"]}>' - ) + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"name": "Jo"} - reference = provider.send_email( - from_address, - recipient, - plain_text_email.subject, - body=str(plain_text_email), - html_body=str(html_email), - reply_to_address=notification.reply_to_text, - ) - notification.reference = reference - update_notification_to_sending(notification, provider) + send_to_providers.send_sms_to_provider(db_notification) + + aws_sns_client.send_sms.assert_called_once_with( + to="2028675309", + content="Sample service: Hello Jo\nHere is some HTML & entities", + reference=str(db_notification.id), + sender=current_app.config["FROM_NUMBER"], + international=False, + ) + + notification = Notification.query.filter_by(id=db_notification.id).one() + + assert notification.status == NotificationStatus.SENDING + assert notification.sent_at <= utc_now() + assert notification.sent_by == "sns" + assert notification.billable_units == 1 + assert notification.personalisation == {"name": "Jo"} -def update_notification_to_sending(notification, provider): - notification.sent_at = utc_now() - notification.sent_by = provider.name - if notification.status not in NotificationStatus.completed_types(): - notification.status = NotificationStatus.SENDING +def test_should_send_personalised_template_to_correct_email_provider_and_persist( + sample_email_template_with_html, mocker +): - dao_update_notification(notification) + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + utf8_encoded_email = "jo.smith@example.com".encode("utf-8") + mock_redis.get.return_value = utf8_encoded_email + email = utf8_encoded_email + personalisation = { + "name": "Jo", + } + personalisation = json.dumps(personalisation) + personalisation = personalisation.encode("utf-8") + mock_redis.get.side_effect = [email, personalisation] + db_notification = create_notification( + template=sample_email_template_with_html, + ) + db_notification.personalisation = {"name": "Jo"} + mocker.patch("app.aws_ses_client.send_email", return_value="reference") + send_to_providers.send_email_to_provider(db_notification) + app.aws_ses_client.send_email.assert_called_once_with( + f'"Sample service" ', + "jo.smith@example.com", + "Jo some HTML", + body="Hello Jo\nThis is an email from GOV.\u200bUK with some HTML\n", + html_body=ANY, + reply_to_address=None, + ) + + assert " version_on_notification + + send_to_providers.send_sms_to_provider(db_notification) + + aws_sns_client.send_sms.assert_called_once_with( + to="2028675309", + content="Sample service: This is a template:\nwith a newline", + reference=str(db_notification.id), + sender=current_app.config["FROM_NUMBER"], + international=False, + ) + + t = dao_get_template_by_id(expected_template_id) + + persisted_notification = notifications_dao.get_notification_by_id( + db_notification.id + ) + assert persisted_notification.to == db_notification.to + assert persisted_notification.template_id == expected_template_id + assert persisted_notification.template_version == version_on_notification + assert persisted_notification.template_version != t.version + assert persisted_notification.status == NotificationStatus.SENDING + + +def test_should_have_sending_status_if_fake_callback_function_fails( + sample_notification, mocker +): + mocker.patch( + "app.delivery.send_to_providers.send_sms_response", + side_effect=HTTPError, + ) + + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "2028675309" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + sample_notification.key_type = KeyType.TEST + with pytest.raises(HTTPError): + send_to_providers.send_sms_to_provider(sample_notification) + assert sample_notification.status == NotificationStatus.SENDING + assert sample_notification.sent_by == "sns" + + +def test_should_not_send_to_provider_when_status_is_not_created( + sample_template, mocker +): + notification = create_notification( + template=sample_template, + status=NotificationStatus.SENDING, + ) + mocker.patch("app.aws_sns_client.send_sms") + response_mock = mocker.patch("app.delivery.send_to_providers.send_sms_response") + + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "2028675309" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider(notification) + + app.aws_sns_client.send_sms.assert_not_called() + response_mock.assert_not_called() + + +def test_should_send_sms_with_downgraded_content(notify_db_session, mocker): + # é, o, and u are in GSM. + # ī, grapes, tabs, zero width space and ellipsis are not + # ó isn't in GSM, but it is in the welsh alphabet so will still be sent + + mocker.patch("app.delivery.send_to_providers.redis_store", return_value=None) + mocker.patch( + "app.delivery.send_to_providers.get_sender_numbers", return_value=["testing"] + ) + msg = "a é ī o u 🍇 foo\tbar\u200bbaz((misc))…" + placeholder = "∆∆∆abc" + gsm_message = "?ódz Housing Service: a é i o u ? foo barbaz???abc..." + service = create_service(service_name="Łódź Housing Service") + template = create_template(service, content=msg) + db_notification = create_notification( + template=template, + ) + db_notification.personalisation = {"misc": placeholder} + db_notification.reply_to_text = "testing" + + mocker.patch("app.aws_sns_client.send_sms") + + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"misc": placeholder} + + send_to_providers.send_sms_to_provider(db_notification) + + aws_sns_client.send_sms.assert_called_once_with( + to=ANY, content=gsm_message, reference=ANY, sender=ANY, international=False + ) + + +def test_send_sms_should_use_service_sms_sender( + sample_service, sample_template, mocker +): + + mocker.patch("app.delivery.send_to_providers.redis_store", return_value=None) + mocker.patch("app.aws_sns_client.send_sms") + + sms_sender = create_service_sms_sender( + service=sample_service, sms_sender="123456", is_default=False + ) + db_notification = create_notification( + template=sample_template, reply_to_text=sms_sender.sms_sender + ) + expected_sender_name = sms_sender.sms_sender + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider( + db_notification, + ) + + app.aws_sns_client.send_sms.assert_called_once_with( + to=ANY, + content=ANY, + reference=ANY, + sender=expected_sender_name, + international=False, + ) + + +def test_send_email_to_provider_should_not_send_to_provider_when_status_is_not_created( + sample_email_template, mocker +): + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + mock_redis.get.return_value = "test@example.com".encode("utf-8") + + notification = create_notification( + template=sample_email_template, status=NotificationStatus.SENDING + ) + mocker.patch("app.aws_ses_client.send_email") + mocker.patch("app.delivery.send_to_providers.send_email_response") + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + send_to_providers.send_sms_to_provider(notification) + app.aws_ses_client.send_email.assert_not_called() + app.delivery.send_to_providers.send_email_response.assert_not_called() + + +def test_send_email_should_use_service_reply_to_email( + sample_service, sample_email_template, mocker +): + mocker.patch("app.aws_ses_client.send_email", return_value="reference") + + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + mock_redis.get.return_value = "test@example.com".encode("utf-8") + + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + email = "foo@bar.com".encode("utf-8") + personalisation = {} + + personalisation = json.dumps(personalisation) + personalisation = personalisation.encode("utf-8") + mock_redis.get.side_effect = [email, personalisation] + + db_notification = create_notification( + template=sample_email_template, reply_to_text="foo@bar.com" + ) + create_reply_to_email(service=sample_service, email_address="foo@bar.com") + + send_to_providers.send_email_to_provider(db_notification) + + app.aws_ses_client.send_email.assert_called_once_with( + ANY, + ANY, + ANY, + body=ANY, + html_body=ANY, + reply_to_address="foo@bar.com", + ) + + +def test_get_html_email_renderer_should_return_for_normal_service(sample_service): + options = send_to_providers.get_html_email_options(sample_service) + assert options["govuk_banner"] is True + assert "brand_colour" not in options.keys() + assert "brand_logo" not in options.keys() + assert "brand_text" not in options.keys() + assert "brand_name" not in options.keys() + + +@pytest.mark.parametrize( + "branding_type, govuk_banner", + [(BrandType.ORG, False), (BrandType.BOTH, True), (BrandType.ORG_BANNER, False)], +) +def test_get_html_email_renderer_with_branding_details( + branding_type, govuk_banner, notify_db_session, sample_service +): + email_branding = EmailBranding( + brand_type=branding_type, + colour="#000000", + logo="justice-league.png", + name="Justice League", + text="League of Justice", + ) + sample_service.email_branding = email_branding + notify_db_session.add_all([sample_service, email_branding]) + notify_db_session.commit() + + options = send_to_providers.get_html_email_options(sample_service) + + assert options["govuk_banner"] == govuk_banner + assert options["brand_colour"] == "#000000" + assert options["brand_text"] == "League of Justice" + assert options["brand_name"] == "Justice League" + + if branding_type == BrandType.ORG_BANNER: + assert options["brand_banner"] is True + else: + assert options["brand_banner"] is False + + +def test_get_html_email_renderer_with_branding_details_and_render_govuk_banner_only( + notify_db_session, sample_service +): + sample_service.email_branding = None + notify_db_session.add_all([sample_service]) + notify_db_session.commit() + + options = send_to_providers.get_html_email_options(sample_service) + + assert options == {"govuk_banner": True, "brand_banner": False} + + +def test_get_html_email_renderer_prepends_logo_path(notify_api): + Service = namedtuple("Service", ["email_branding"]) + EmailBranding = namedtuple( + "EmailBranding", + ["brand_type", "colour", "name", "logo", "text"], + ) + + email_branding = EmailBranding( + brand_type=BrandType.ORG, + colour="#000000", + logo="justice-league.png", + name="Justice League", + text="League of Justice", + ) + service = Service( + email_branding=email_branding, + ) + + renderer = send_to_providers.get_html_email_options(service) + + assert ( + renderer["brand_logo"] == "http://static-logos.notify.tools/justice-league.png" + ) + + +def test_get_html_email_renderer_handles_email_branding_without_logo(notify_api): + Service = namedtuple("Service", ["email_branding"]) + EmailBranding = namedtuple( + "EmailBranding", + ["brand_type", "colour", "name", "logo", "text"], + ) + + email_branding = EmailBranding( + brand_type=BrandType.ORG_BANNER, + colour="#000000", + logo=None, + name="Justice League", + text="League of Justice", + ) + service = Service( + email_branding=email_branding, + ) + + renderer = send_to_providers.get_html_email_options(service) + + assert renderer["govuk_banner"] is False + assert renderer["brand_banner"] is True + assert renderer["brand_logo"] is None + assert renderer["brand_text"] == "League of Justice" + assert renderer["brand_colour"] == "#000000" + assert renderer["brand_name"] == "Justice League" + + +@pytest.mark.parametrize( + "base_url, expected_url", + [ + # don't change localhost to prevent errors when testing locally + ("http://localhost:6012", "http://static-logos.notify.tools/filename.png"), + ( + "https://www.notifications.service.gov.uk", + "https://static-logos.notifications.service.gov.uk/filename.png", + ), + ("https://notify.works", "https://static-logos.notify.works/filename.png"), + ( + "https://staging-notify.works", + "https://static-logos.staging-notify.works/filename.png", + ), + ("https://www.notify.works", "https://static-logos.notify.works/filename.png"), + ( + "https://www.staging-notify.works", + "https://static-logos.staging-notify.works/filename.png", + ), + ], +) +def test_get_logo_url_works_for_different_environments(base_url, expected_url): + logo_file = "filename.png" + + logo_url = send_to_providers.get_logo_url(base_url, logo_file) + + assert logo_url == expected_url + + +@pytest.mark.parametrize( + "starting_status, expected_status", + [ + (NotificationStatus.DELIVERED, NotificationStatus.DELIVERED), + (NotificationStatus.CREATED, NotificationStatus.SENDING), + (NotificationStatus.TECHNICAL_FAILURE, NotificationStatus.TECHNICAL_FAILURE), + ], +) +def test_update_notification_to_sending_does_not_update_status_from_a_final_status( + sample_service, notify_db_session, starting_status, expected_status +): + template = create_template(sample_service) + notification = create_notification(template=template, status=starting_status) + send_to_providers.update_notification_to_sending( + notification, + notification_provider_clients.get_client_by_name_and_type( + "sns", NotificationType.SMS + ), + ) + assert notification.status == expected_status + + +def __update_notification(notification_to_update, research_mode, expected_status): + if research_mode or notification_to_update.key_type == KeyType.TEST: + notification_to_update.status = expected_status + + +@pytest.mark.parametrize( + "research_mode,key_type, billable_units, expected_status", + [ + (True, KeyType.NORMAL, 0, NotificationStatus.DELIVERED), + (False, KeyType.NORMAL, 1, NotificationStatus.SENDING), + (False, KeyType.TEST, 0, NotificationStatus.SENDING), + (True, KeyType.TEST, 0, NotificationStatus.SENDING), + (True, KeyType.TEAM, 0, NotificationStatus.DELIVERED), + (False, KeyType.TEAM, 1, NotificationStatus.SENDING), + ], +) +def test_should_update_billable_units_and_status_according_to_research_mode_and_key_type( + sample_template, mocker, research_mode, key_type, billable_units, expected_status +): + + mocker.patch("app.delivery.send_to_providers.redis_store", return_value=None) + mocker.patch( + "app.delivery.send_to_providers.get_sender_numbers", return_value=["testing"] + ) + notification = create_notification( + template=sample_template, + billable_units=0, + status=NotificationStatus.CREATED, + key_type=key_type, + reply_to_text="testing", + ) + mocker.patch("app.aws_sns_client.send_sms") + mocker.patch( + "app.delivery.send_to_providers.send_sms_response", + side_effect=__update_notification(notification, research_mode, expected_status), + ) + + if research_mode: + sample_template.service.research_mode = True + + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + # So we don't treat it as a one off and have to mock other things + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider(notification) + assert notification.billable_units == billable_units + assert notification.status == expected_status + + +def test_should_set_notification_billable_units_and_reduces_provider_priority_if_sending_to_provider_fails( + sample_notification, + mocker, +): + mocker.patch("app.aws_sns_client.send_sms", side_effect=Exception()) + + sample_notification.billable_units = 0 + assert sample_notification.sent_by is None + + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + # flake8 no longer likes raises with a generic exception + try: + send_to_providers.send_sms_to_provider(sample_notification) + assert 1 == 0 + except Exception: + assert 1 == 1 + + assert sample_notification.billable_units == 1 + + +def test_should_send_sms_to_international_providers( + sample_template, sample_user, mocker +): + + mocker.patch("app.delivery.send_to_providers._get_verify_code", return_value=None) + mocker.patch("app.aws_sns_client.send_sms") + + notification_international = create_notification( + template=sample_template, + to_field="+6011-17224412", + personalisation={"name": "Jo"}, + status=NotificationStatus.CREATED, + international=True, + reply_to_text=sample_template.service.get_default_sms_sender(), + normalised_to="601117224412", + ) + + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "601117224412" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider(notification_international) + + aws_sns_client.send_sms.assert_called_once_with( + to="601117224412", + content=ANY, + reference=str(notification_international.id), + sender=current_app.config["FROM_NUMBER"], + international=True, + ) + + assert notification_international.status == NotificationStatus.SENDING + assert notification_international.sent_by == "sns" + + +@pytest.mark.parametrize( + "sms_sender, expected_sender, prefix_sms, expected_content", + [ + ("foo", "foo", False, "bar"), + ("foo", "foo", True, "Sample service: bar"), + # if 40604 is actually in DB then treat that as if entered manually + ("40604", "40604", False, "bar"), + # 'testing' is the FROM_NUMBER during unit tests + ("testing", "testing", True, "Sample service: bar"), + ("testing", "testing", False, "bar"), + ], +) +def test_should_handle_sms_sender_and_prefix_message( + mocker, sms_sender, prefix_sms, expected_sender, expected_content, notify_db_session +): + + mocker.patch("app.delivery.send_to_providers.redis_store", return_value=None) + mocker.patch("app.aws_sns_client.send_sms") + service = create_service_with_defined_sms_sender( + sms_sender_value=sms_sender, prefix_sms=prefix_sms + ) + template = create_template(service, content="bar") + notification = create_notification(template, reply_to_text=sms_sender) + + mock_phone = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_phone.return_value = "15555555555" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider(notification) + + aws_sns_client.send_sms.assert_called_once_with( + content=expected_content, + sender=expected_sender, + to=ANY, + reference=ANY, + international=False, + ) + + +def test_send_email_to_provider_uses_reply_to_from_notification( + sample_email_template, mocker +): + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + mock_redis.get.side_effect = [ + "test@example.com".encode("utf-8"), + json.dumps({}).encode("utf-8"), ] - if not active_providers: - current_app.logger.error(f"{notification_type} failed as no active providers") - raise Exception(f"No active {notification_type} providers") + mocker.patch("app.aws_ses_client.send_email", return_value="reference") - # we only have sns - chosen_provider = active_providers[0] + db_notification = create_notification( + template=sample_email_template, + reply_to_text="test@test.com", + ) - return notification_provider_clients.get_client_by_name_and_type( - chosen_provider.identifier, notification_type + send_to_providers.send_email_to_provider(db_notification) + + app.aws_ses_client.send_email.assert_called_once_with( + ANY, + ANY, + ANY, + body=ANY, + html_body=ANY, + reply_to_address="test@test.com", ) -def get_logo_url(base_url, logo_file): - base_url = parse.urlparse(base_url) - netloc = base_url.netloc +def test_send_sms_to_provider_should_use_normalised_to(mocker, client, sample_template): - if base_url.netloc.startswith("localhost"): - netloc = "notify.tools" - elif base_url.netloc.startswith("www"): - # strip "www." - netloc = base_url.netloc[4:] - - logo_url = parse.ParseResult( - scheme=base_url.scheme, - netloc="static-logos." + netloc, - path=logo_file, - params=base_url.params, - query=base_url.query, - fragment=base_url.fragment, + mocker.patch("app.delivery.send_to_providers._get_verify_code", return_value=None) + mocker.patch( + "app.delivery.send_to_providers.get_sender_numbers", return_value=["testing"] ) - return parse.urlunparse(logo_url) - - -def get_html_email_options(service): - if service.email_branding is None: - return { - "govuk_banner": True, - "brand_banner": False, - } - if isinstance(service, SerialisedService): - branding = dao_get_email_branding_by_id(service.email_branding) - else: - branding = service.email_branding - - logo_url = ( - get_logo_url(current_app.config["ADMIN_BASE_URL"], branding.logo) - if branding.logo - else None + send_mock = mocker.patch("app.aws_sns_client.send_sms") + notification = create_notification( + template=sample_template, + to_field="+12028675309", + normalised_to="2028675309", + reply_to_text="testing", ) - return { + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "12028675309" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + send_to_providers.send_sms_to_provider(notification) + send_mock.assert_called_once_with( + to="12028675309", + content=ANY, + reference=str(notification.id), + sender=notification.reply_to_text, + international=False, + ) + + +def test_send_email_to_provider_should_user_normalised_to( + mocker, client, sample_email_template +): + send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference") + notification = create_notification( + template=sample_email_template, + ) + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + mock_redis.get.return_value = "test@example.com".encode("utf-8") + + mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + mock_redis.get.return_value = "jo.smith@example.com".encode("utf-8") + email = "test@example.com".encode("utf-8") + personalisation = {} + + personalisation = json.dumps(personalisation) + personalisation = personalisation.encode("utf-8") + mock_redis.get.side_effect = [email, personalisation] + + send_to_providers.send_email_to_provider(notification) + send_mock.assert_called_once_with( + ANY, + "test@example.com", + ANY, + body=ANY, + html_body=ANY, + reply_to_address=notification.reply_to_text, + ) + + +def test_send_sms_to_provider_should_return_template_if_found_in_redis( + mocker, client, sample_template +): + + mocker.patch("app.delivery.send_to_providers._get_verify_code", return_value=None) + mocker.patch( + "app.delivery.send_to_providers.get_sender_numbers", return_value=["testing"] + ) + from app.schemas import service_schema, template_schema + + service_dict = service_schema.dump(sample_template.service) + template_dict = template_schema.dump(sample_template) + + mocker.patch( + "app.redis_store.get", + side_effect=[ + json.dumps({"data": service_dict}).encode("utf-8"), + json.dumps({"data": template_dict}).encode("utf-8"), + ], + ) + mock_get_template = mocker.patch( + "app.dao.templates_dao.dao_get_template_by_id_and_service_id" + ) + mock_get_service = mocker.patch("app.dao.services_dao.dao_fetch_service_by_id") + + send_mock = mocker.patch("app.aws_sns_client.send_sms") + notification = create_notification( + template=sample_template, + to_field="+447700900855", + normalised_to="447700900855", + reply_to_text="testing", + ) + + mock_s3 = mocker.patch("app.delivery.send_to_providers.get_phone_number_from_s3") + mock_s3.return_value = "447700900855" + + mock_personalisation = mocker.patch( + "app.delivery.send_to_providers.get_personalisation_from_s3" + ) + mock_personalisation.return_value = {"ignore": "ignore"} + + send_to_providers.send_sms_to_provider(notification) + assert mock_get_template.called is False + assert mock_get_service.called is False + send_mock.assert_called_once_with( + to="447700900855", + content=ANY, + reference=str(notification.id), + sender=notification.reply_to_text, + international=False, + ) + + +def test_send_email_to_provider_should_return_template_if_found_in_redis( + mocker, client, sample_email_template +): + from app.schemas import service_schema, template_schema + + # mock_redis = mocker.patch("app.delivery.send_to_providers.redis_store") + # mock_redis.get.return_value = "jo.smith@example.com".encode("utf-8") + email = "test@example.com".encode("utf-8") + personalisation = { + "name": "Jo", + } + + personalisation = json.dumps(personalisation) + personalisation = personalisation.encode("utf-8") + # mock_redis.get.side_effect = [email, personalisation] + + service_dict = service_schema.dump(sample_email_template.service) + template_dict = template_schema.dump(sample_email_template) + + mocker.patch( + "app.redis_store.get", + side_effect=[ + email, + personalisation, + json.dumps({"data": service_dict}).encode("utf-8"), + json.dumps({"data": template_dict}).encode("utf-8"), + ], + ) + mock_get_template = mocker.patch( + "app.dao.templates_dao.dao_get_template_by_id_and_service_id" + ) + mock_get_service = mocker.patch("app.dao.services_dao.dao_fetch_service_by_id") + send_mock = mocker.patch("app.aws_ses_client.send_email", return_value="reference") + notification = create_notification( + template=sample_email_template, + ) + + send_to_providers.send_email_to_provider(notification) + assert mock_get_template.called is False + assert mock_get_service.called is False + send_mock.assert_called_once_with( + ANY, + "test@example.com", + ANY, + body=ANY, + html_body=ANY, + reply_to_address=notification.reply_to_text, + ) + + +def test_get_html_email_options_return_email_branding_from_serialised_service( + sample_service, +): + branding = create_email_branding() + sample_service.email_branding = branding + service = SerialisedService.from_id(sample_service.id) + email_options = get_html_email_options(service) + assert email_options is not None + assert email_options == { "govuk_banner": branding.brand_type == BrandType.BOTH, "brand_banner": branding.brand_type == BrandType.ORG_BANNER, "brand_colour": branding.colour, - "brand_logo": logo_url, + "brand_logo": get_logo_url(current_app.config["ADMIN_BASE_URL"], branding.logo), "brand_text": branding.text, "brand_name": branding.name, } -def technical_failure(notification): - notification.status = NotificationStatus.TECHNICAL_FAILURE - dao_update_notification(notification) - raise NotificationTechnicalFailureException( - f"Send {notification.notification_type} for notification id {notification.id} " - f"to provider is not allowed: service {notification.service_id} is inactive" - ) +def test_get_html_email_options_add_email_branding_from_service(sample_service): + branding = create_email_branding() + sample_service.email_branding = branding + email_options = get_html_email_options(sample_service) + assert email_options is not None + assert email_options == { + "govuk_banner": branding.brand_type == BrandType.BOTH, + "brand_banner": branding.brand_type == BrandType.ORG_BANNER, + "brand_colour": branding.colour, + "brand_logo": get_logo_url(current_app.config["ADMIN_BASE_URL"], branding.logo), + "brand_text": branding.text, + "brand_name": branding.name, + } @pytest.mark.parametrize( From 05a6a2a4d9f69f52fa40cb5eea43684730bd180a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 14:04:26 -0700 Subject: [PATCH 005/128] comment out strange command we may never use --- app/commands.py | 171 ++++++++++++++++++++++++------------------------ 1 file changed, 84 insertions(+), 87 deletions(-) diff --git a/app/commands.py b/app/commands.py index 45fce9211..1c761f84a 100644 --- a/app/commands.py +++ b/app/commands.py @@ -24,12 +24,6 @@ from app.dao.annual_billing_dao import ( dao_create_or_update_annual_billing_for_year, set_default_free_allowance_for_service, ) -from app.dao.fact_billing_dao import ( - delete_billing_data_for_service_for_day, - fetch_billing_data_for_day, - get_service_ids_that_need_billing_populated, - update_fact_billing, -) from app.dao.jobs_dao import dao_get_job_by_id from app.dao.organization_dao import ( dao_add_service_to_organization, @@ -63,7 +57,7 @@ from app.models import ( TemplateHistory, User, ) -from app.utils import get_midnight_in_utc, utc_now +from app.utils import utc_now from notifications_utils.recipients import RecipientCSV from notifications_utils.template import SMSMessageTemplate from tests.app.db import ( @@ -167,76 +161,78 @@ def purge_functional_test_data(user_email_prefix): delete_model_user(usr) -@notify_command(name="insert-inbound-numbers") -@click.option( - "-f", - "--file_name", - required=True, - help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""", -) -def insert_inbound_numbers_from_file(file_name): - # TODO maintainability what is the purpose of this command? Who would use it and why? +# TODO maintainability what is the purpose of this command? Who would use it and why? +# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT +# @notify_command(name="insert-inbound-numbers") +# @click.option( +# "-f", +# "--file_name", +# required=True, +# help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""", +# ) +# def insert_inbound_numbers_from_file(file_name): - current_app.logger.info(f"Inserting inbound numbers from {file_name}") - with open(file_name) as file: - sql = text( - "insert into inbound_numbers values(:uuid, :line, 'sns', null, True, now(), null);" - ) +# current_app.logger.info(f"Inserting inbound numbers from {file_name}") +# with open(file_name) as file: +# sql = text( +# "insert into inbound_numbers values(:uuid, :line, 'sns', null, True, now(), null);" +# ) - for line in file: - line = line.strip() - if line: - current_app.logger.info(line) - db.session.execute(sql, {"uuid": str(uuid.uuid4()), "line": line}) - db.session.commit() +# for line in file: +# line = line.strip() +# if line: +# current_app.logger.info(line) +# db.session.execute(sql, {"uuid": str(uuid.uuid4()), "line": line}) +# db.session.commit() def setup_commands(application): application.cli.add_command(command_group) -@notify_command(name="rebuild-ft-billing-for-day") -@click.option("-s", "--service_id", required=False, type=click.UUID) -@click.option( - "-d", - "--day", - help="The date to recalculate, as YYYY-MM-DD", - required=True, - type=click_dt(format="%Y-%m-%d"), -) -def rebuild_ft_billing_for_day(service_id, day): - # TODO maintainability what is the purpose of this command? Who would use it and why? +# TODO maintainability what is the purpose of this command? Who would use it and why? +# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT +# @notify_command(name="rebuild-ft-billing-for-day") +# @click.option("-s", "--service_id", required=False, type=click.UUID) +# @click.option( +# "-d", +# "--day", +# help="The date to recalculate, as YYYY-MM-DD", +# required=True, +# type=click_dt(format="%Y-%m-%d"), +# ) +# def rebuild_ft_billing_for_day(service_id, day): - """ - Rebuild the data in ft_billing for the given service_id and date - """ +# """ +# Rebuild the data in ft_billing for the given service_id and date +# """ - def rebuild_ft_data(process_day, service): - deleted_rows = delete_billing_data_for_service_for_day(process_day, service) - current_app.logger.info( - f"deleted {deleted_rows} existing billing rows for {service} on {process_day}" - ) - transit_data = fetch_billing_data_for_day( - process_day=process_day, service_id=service - ) - # transit_data = every row that should exist - for data in transit_data: - # upsert existing rows - update_fact_billing(data, process_day) - current_app.logger.info( - f"added/updated {len(transit_data)} billing rows for {service} on {process_day}" - ) +# def rebuild_ft_data(process_day, service): +# deleted_rows = delete_billing_data_for_service_for_day(process_day, service) +# current_app.logger.info( +# f"deleted {deleted_rows} existing billing rows for {service} on {process_day}" +# ) +# transit_data = fetch_billing_data_for_day( +# process_day=process_day, service_id=service +# ) +# # transit_data = every row that should exist +# for data in transit_data: +# # upsert existing rows +# update_fact_billing(data, process_day) +# current_app.logger.info( +# f"added/updated {len(transit_data)} billing rows for {service} on {process_day}" +# ) - if service_id: - # confirm the service exists - dao_fetch_service_by_id(service_id) - rebuild_ft_data(day, service_id) - else: - services = get_service_ids_that_need_billing_populated( - get_midnight_in_utc(day), get_midnight_in_utc(day + timedelta(days=1)) - ) - for row in services: - rebuild_ft_data(day, row.service_id) +# if service_id: +# # confirm the service exists +# dao_fetch_service_by_id(service_id) +# rebuild_ft_data(day, service_id) +# else: +# services = get_service_ids_that_need_billing_populated( +# get_midnight_in_utc(day), get_midnight_in_utc(day + timedelta(days=1)) +# ) +# for row in services: +# rebuild_ft_data(day, row.service_id) @notify_command(name="bulk-invite-user-to-service") @@ -472,29 +468,30 @@ def associate_services_to_organizations(): current_app.logger.info("finished associating services to organizations") -@notify_command(name="populate-service-volume-intentions") -@click.option( - "-f", - "--file_name", - required=True, - help="Pipe delimited file containing service_id, SMS, email", -) -def populate_service_volume_intentions(file_name): - # [0] service_id - # [1] SMS:: volume intentions for service - # [2] Email:: volume intentions for service +# TODO maintainability what is the purpose of this command? Who would use it and why? +# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT +# @notify_command(name="populate-service-volume-intentions") +# @click.option( +# "-f", +# "--file_name", +# required=True, +# help="Pipe delimited file containing service_id, SMS, email", +# ) +# def populate_service_volume_intentions(file_name): +# # [0] service_id +# # [1] SMS:: volume intentions for service +# # [2] Email:: volume intentions for service - # TODO maintainability what is the purpose of this command? Who would use it and why? - with open(file_name, "r") as f: - for line in itertools.islice(f, 1, None): - columns = line.split(",") - current_app.logger.info(columns) - service = dao_fetch_service_by_id(columns[0]) - service.volume_sms = columns[1] - service.volume_email = columns[2] - dao_update_service(service) - current_app.logger.info("populate-service-volume-intentions complete") +# with open(file_name, "r") as f: +# for line in itertools.islice(f, 1, None): +# columns = line.split(",") +# current_app.logger.info(columns) +# service = dao_fetch_service_by_id(columns[0]) +# service.volume_sms = columns[1] +# service.volume_email = columns[2] +# dao_update_service(service) +# current_app.logger.info("populate-service-volume-intentions complete") @notify_command(name="populate-go-live") From 06643c3bb50dc58963fa7dae7139843aa5fdb862 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 14:11:47 -0700 Subject: [PATCH 006/128] comment out strange command we may never use --- app/commands.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/app/commands.py b/app/commands.py index 1c761f84a..a43ae06ca 100644 --- a/app/commands.py +++ b/app/commands.py @@ -162,28 +162,27 @@ def purge_functional_test_data(user_email_prefix): # TODO maintainability what is the purpose of this command? Who would use it and why? -# COMMENTING OUT UNTIL WE DETERMINE IF WE NEED IT OR NOT -# @notify_command(name="insert-inbound-numbers") -# @click.option( -# "-f", -# "--file_name", -# required=True, -# help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""", -# ) -# def insert_inbound_numbers_from_file(file_name): +@notify_command(name="insert-inbound-numbers") +@click.option( + "-f", + "--file_name", + required=True, + help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""", +) +def insert_inbound_numbers_from_file(file_name): -# current_app.logger.info(f"Inserting inbound numbers from {file_name}") -# with open(file_name) as file: -# sql = text( -# "insert into inbound_numbers values(:uuid, :line, 'sns', null, True, now(), null);" -# ) + current_app.logger.info(f"Inserting inbound numbers from {file_name}") + with open(file_name) as file: + sql = text( + "insert into inbound_numbers values(:uuid, :line, 'sns', null, True, now(), null);" + ) -# for line in file: -# line = line.strip() -# if line: -# current_app.logger.info(line) -# db.session.execute(sql, {"uuid": str(uuid.uuid4()), "line": line}) -# db.session.commit() + for line in file: + line = line.strip() + if line: + current_app.logger.info(line) + db.session.execute(sql, {"uuid": str(uuid.uuid4()), "line": line}) + db.session.commit() def setup_commands(application): From face881a90ecb0ab40f3ee535080e8bd71bb6b1c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 14:45:22 -0700 Subject: [PATCH 007/128] clean up sanitise_text --- notifications_utils/sanitise_text.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/notifications_utils/sanitise_text.py b/notifications_utils/sanitise_text.py index 3e9da0764..5a1d1c382 100644 --- a/notifications_utils/sanitise_text.py +++ b/notifications_utils/sanitise_text.py @@ -122,19 +122,15 @@ class SanitiseText: def is_punjabi(cls, value): # Gukmukhi script or Shahmukhi script - if regex.search(r"[\u0A00-\u0A7F]+", value): - return True - elif regex.search(r"[\u0600-\u06FF]+", value): - return True - elif regex.search(r"[\u0750-\u077F]+", value): - return True - elif regex.search(r"[\u08A0-\u08FF]+", value): - return True - elif regex.search(r"[\uFB50-\uFDFF]+", value): - return True - elif regex.search(r"[\uFE70-\uFEFF]+", value): - return True - elif regex.search(r"[\u0900-\u097F]+", value): + if ( + regex.search(r"[\u0A00-\u0A7F]+", value) + or regex.search(r"[\u0600-\u06FF]+", value) + or regex.search(r"[\u0750-\u077F]+", value) + or regex.search(r"[\u08A0-\u08FF]+", value) + or regex.search(r"[\uFB50-\uFDFF]+", value) + or regex.search(r"[\uFE70-\uFEFF]+", value) + or regex.search(r"[\u0900-\u097F]+", value) + ): return True return False From dba29a8ea7fff2d5f651c3669462dcc8bc3e9115 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 14:55:15 -0700 Subject: [PATCH 008/128] clean up sanitise_text --- notifications_utils/sanitise_text.py | 38 ++++++++++++---------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/notifications_utils/sanitise_text.py b/notifications_utils/sanitise_text.py index 5a1d1c382..750a2e49b 100644 --- a/notifications_utils/sanitise_text.py +++ b/notifications_utils/sanitise_text.py @@ -152,33 +152,27 @@ class SanitiseText: @classmethod def _is_extended_language_group_two(cls, value): - if regex.search(r"\p{IsBuhid}", value): - return True - if regex.search(r"\p{IsCanadian_Aboriginal}", value): - return True - if regex.search(r"\p{IsCherokee}", value): - return True - if regex.search(r"\p{IsDevanagari}", value): - return True - if regex.search(r"\p{IsEthiopic}", value): - return True - if regex.search(r"\p{IsGeorgian}", value): + if ( + regex.search(r"\p{IsBuhid}", value) + or regex.search(r"\p{IsCanadian_Aboriginal}", value) + or regex.search(r"\p{IsCherokee}", value) + or regex.search(r"\p{IsDevanagari}", value) + or regex.search(r"\p{IsEthiopic}", value) + or regex.search(r"\p{IsGeorgian}", value) + ): return True return False @classmethod def _is_extended_language_group_three(cls, value): - if regex.search(r"\p{IsGreek}", value): - return True - if regex.search(r"\p{IsGujarati}", value): - return True - if regex.search(r"\p{IsHanunoo}", value): - return True - if regex.search(r"\p{IsHebrew}", value): - return True - if regex.search(r"\p{IsLimbu}", value): - return True - if regex.search(r"\p{IsKannada}", value): + if ( + regex.search(r"\p{IsGreek}", value) + or regex.search(r"\p{IsGujarati}", value) + or regex.search(r"\p{IsHanunoo}", value) + or regex.search(r"\p{IsHebrew}", value) + or regex.search(r"\p{IsLimbu}", value) + or regex.search(r"\p{IsKannada}", value) + ): return True return False From 445a462b1052353e85475b284fb93c69db71650f Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 2 Oct 2024 15:00:03 -0700 Subject: [PATCH 009/128] clean up s3 --- app/aws/s3.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index bd0301d78..dc293ea6f 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -466,23 +466,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number): set_job_cache(job_cache, f"{job_id}_personalisation", extract_personalisation(job)) - # If we can find the quick dictionary, use it - if job_cache.get(f"{job_id}_personalisation") is not None: - personalisation_to_return = job_cache.get(f"{job_id}_personalisation")[0].get( - job_row_number - ) - if personalisation_to_return: - return personalisation_to_return - else: - current_app.logger.warning( - f"Was unable to retrieve personalisation from lookup dictionary for job {job_id}" - ) - return {} - else: - current_app.logger.error( - f"Was unable to construct lookup dictionary for job {job_id}" - ) - return {} + return job_cache.get(f"{job_id}_personalisation")[0].get(job_row_number) def get_job_metadata_from_s3(service_id, job_id): From 76373de13b9f05b4b71c37d5206125af480c066e Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 08:54:16 -0700 Subject: [PATCH 010/128] comment out search for notification by to field --- app/service/rest.py | 106 ++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/app/service/rest.py b/app/service/rest.py index 070f13457..9ae507adb 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -453,16 +453,16 @@ def get_all_notifications_for_service(service_id): data = notifications_filter_schema.load(MultiDict(request.get_json())) current_app.logger.debug(f"use POST, request {request.get_json()} data {data}") - if data.get("to"): - notification_type = ( - data.get("template_type")[0] if data.get("template_type") else None - ) - return search_for_notification_by_to_field( - service_id=service_id, - search_term=data["to"], - statuses=data.get("status"), - notification_type=notification_type, - ) + # if data.get("to"): + # notification_type = ( + # data.get("template_type")[0] if data.get("template_type") else None + # ) + # return search_for_notification_by_to_field( + # service_id=service_id, + # search_term=data["to"], + # statuses=data.get("status"), + # notification_type=notification_type, + # ) page = data["page"] if "page" in data else 1 page_size = ( data["page_size"] @@ -583,51 +583,51 @@ def get_notification_for_service(service_id, notification_id): ) -def search_for_notification_by_to_field( - service_id, search_term, statuses, notification_type -): - results = notifications_dao.dao_get_notifications_by_recipient_or_reference( - service_id=service_id, - search_term=search_term, - statuses=statuses, - notification_type=notification_type, - page=1, - page_size=current_app.config["PAGE_SIZE"], - ) +# def search_for_notification_by_to_field( +# service_id, search_term, statuses, notification_type +# ): +# results = notifications_dao.dao_get_notifications_by_recipient_or_reference( +# service_id=service_id, +# search_term=search_term, +# statuses=statuses, +# notification_type=notification_type, +# page=1, +# page_size=current_app.config["PAGE_SIZE"], +# ) - # We try and get the next page of results to work out if we need provide a pagination link to the next page - # in our response. Note, this was previously be done by having - # notifications_dao.dao_get_notifications_by_recipient_or_reference use count=False when calling - # Flask-Sqlalchemys `paginate'. But instead we now use this way because it is much more performant for - # services with many results (unlike using Flask SqlAlchemy `paginate` with `count=True`, this approach - # doesn't do an additional query to count all the results of which there could be millions but instead only - # asks for a single extra page of results). - next_page_of_pagination = notifications_dao.dao_get_notifications_by_recipient_or_reference( - service_id=service_id, - search_term=search_term, - statuses=statuses, - notification_type=notification_type, - page=2, - page_size=current_app.config["PAGE_SIZE"], - error_out=False, # False so that if there are no results, it doesn't end in aborting with a 404 - ) +# # We try and get the next page of results to work out if we need provide a pagination link to the next page +# # in our response. Note, this was previously be done by having +# # notifications_dao.dao_get_notifications_by_recipient_or_reference use count=False when calling +# # Flask-Sqlalchemys `paginate'. But instead we now use this way because it is much more performant for +# # services with many results (unlike using Flask SqlAlchemy `paginate` with `count=True`, this approach +# # doesn't do an additional query to count all the results of which there could be millions but instead only +# # asks for a single extra page of results). +# next_page_of_pagination = notifications_dao.dao_get_notifications_by_recipient_or_reference( +# service_id=service_id, +# search_term=search_term, +# statuses=statuses, +# notification_type=notification_type, +# page=2, +# page_size=current_app.config["PAGE_SIZE"], +# error_out=False, # False so that if there are no results, it doesn't end in aborting with a 404 +# ) - return ( - jsonify( - notifications=notification_with_template_schema.dump( - results.items, many=True - ), - links=get_prev_next_pagination_links( - 1, - len(next_page_of_pagination.items), - ".get_all_notifications_for_service", - statuses=statuses, - notification_type=notification_type, - service_id=service_id, - ), - ), - 200, - ) +# return ( +# jsonify( +# notifications=notification_with_template_schema.dump( +# results.items, many=True +# ), +# links=get_prev_next_pagination_links( +# 1, +# len(next_page_of_pagination.items), +# ".get_all_notifications_for_service", +# statuses=statuses, +# notification_type=notification_type, +# service_id=service_id, +# ), +# ), +# 200, +# ) @service_blueprint.route("//notifications/monthly", methods=["GET"]) From 38583c28eaa99ba081a58baa196de5e01fc8ae15 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 09:23:16 -0700 Subject: [PATCH 011/128] add a test in service rest --- tests/app/service/test_rest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index fec71cf82..5e179b708 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1959,6 +1959,29 @@ def test_get_all_notifications_for_service_including_ones_made_by_jobs( assert response.status_code == 200 +def test_get_monthly_notification_stats_by_user( + client, + sample_service, + sample_user, + mocker, +): + mock_s3 = mocker.patch("app.service.rest.get_phone_number_from_s3") + mock_s3.return_value = "" + + mock_s3 = mocker.patch("app.service.rest.get_personalisation_from_s3") + mock_s3.return_value = {} + + auth_header = create_admin_authorization_header() + + response = client.get( + path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/monthly"), + headers=[auth_header], + ) + + # TODO This test could be a little more complete + assert response.status_code == 200 + + def test_get_only_api_created_notifications_for_service( admin_request, sample_job, From 55966267c2d13bd861885c3bf1b33717a19bced1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 09:35:05 -0700 Subject: [PATCH 012/128] add a test in service rest --- tests/app/service/test_rest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 5e179b708..d6f87d0f6 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1975,9 +1975,12 @@ def test_get_monthly_notification_stats_by_user( response = client.get( path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/monthly"), + year=2024, headers=[auth_header], ) + resp = json.loads(response.get_data(as_text=True)) + print(f"RESP is {resp}") # TODO This test could be a little more complete assert response.status_code == 200 From b0735ffcdce7808e2be10bf3ddda2dcc0798871a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 09:48:32 -0700 Subject: [PATCH 013/128] add a test in service rest --- tests/app/service/test_rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index d6f87d0f6..1c5170596 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1975,8 +1975,8 @@ def test_get_monthly_notification_stats_by_user( response = client.get( path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/monthly"), - year=2024, headers=[auth_header], + year="2024", ) resp = json.loads(response.get_data(as_text=True)) From 5277f7066035b6f7e0c444f94d3083f8b2b820b1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 09:59:41 -0700 Subject: [PATCH 014/128] add a test in service rest --- tests/app/service/test_rest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 1c5170596..d17f778de 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1974,9 +1974,8 @@ def test_get_monthly_notification_stats_by_user( auth_header = create_admin_authorization_header() response = client.get( - path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/monthly"), + path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/monthly?year=2024"), headers=[auth_header], - year="2024", ) resp = json.loads(response.get_data(as_text=True)) From 6d05c1a18ba39308bbe7d4290959608c42565f39 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 10:13:37 -0700 Subject: [PATCH 015/128] add a test in service rest --- tests/app/service/test_rest.py | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index d17f778de..70104930f 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1984,6 +1984,55 @@ def test_get_monthly_notification_stats_by_user( assert response.status_code == 200 +def test_get_single_month_notification_stats_by_user( + client, + sample_service, + sample_user, + mocker, +): + mock_s3 = mocker.patch("app.service.rest.get_phone_number_from_s3") + mock_s3.return_value = "" + + mock_s3 = mocker.patch("app.service.rest.get_personalisation_from_s3") + mock_s3.return_value = {} + + auth_header = create_admin_authorization_header() + + response = client.get( + path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/month/?year=2024&month=07"), + headers=[auth_header], + ) + + resp = json.loads(response.get_data(as_text=True)) + print(f"RESP is {resp}") + # TODO This test could be a little more complete + assert response.status_code == 200 + + +def test_get_single_month_notification_stats_for_service( + client, + sample_service, + mocker, +): + mock_s3 = mocker.patch("app.service.rest.get_phone_number_from_s3") + mock_s3.return_value = "" + + mock_s3 = mocker.patch("app.service.rest.get_personalisation_from_s3") + mock_s3.return_value = {} + + auth_header = create_admin_authorization_header() + + response = client.get( + path=(f"/service/{sample_service.id}/notifications/month/?year=2024&month=07"), + headers=[auth_header], + ) + + resp = json.loads(response.get_data(as_text=True)) + print(f"RESP is {resp}") + # TODO This test could be a little more complete + assert response.status_code == 200 + + def test_get_only_api_created_notifications_for_service( admin_request, sample_job, From 1c95cd63e76afbb0f2b80c42014a9b778ac84b41 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 10:22:32 -0700 Subject: [PATCH 016/128] add a test in service rest --- tests/app/service/test_rest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 70104930f..5ea6e1168 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1999,7 +1999,7 @@ def test_get_single_month_notification_stats_by_user( auth_header = create_admin_authorization_header() response = client.get( - path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/month/?year=2024&month=07"), + path=(f"/service/{sample_service.id}/notifications/{sample_user.id}/month?year=2024&month=07"), headers=[auth_header], ) @@ -2023,7 +2023,7 @@ def test_get_single_month_notification_stats_for_service( auth_header = create_admin_authorization_header() response = client.get( - path=(f"/service/{sample_service.id}/notifications/month/?year=2024&month=07"), + path=(f"/service/{sample_service.id}/notifications/month?year=2024&month=07"), headers=[auth_header], ) From ff5d405a1528f3027e0de6854fff52528b974a80 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 3 Oct 2024 10:30:30 -0700 Subject: [PATCH 017/128] raise coverage to 93 --- .github/workflows/checks.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 22c7f9c89..bcf0861e4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -63,7 +63,7 @@ jobs: NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} - name: Check coverage threshold # TODO get this back up to 95 - run: poetry run coverage report -m --fail-under=91 + run: poetry run coverage report -m --fail-under=93 validate-new-relic-config: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 88cf6f814..76c38d94e 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ test: ## Run tests and create coverage report poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10 ## TODO set this back to 95 asap - poetry run coverage report -m --fail-under=91 + poetry run coverage report -m --fail-under=93 poetry run coverage html -d .coverage_cache .PHONY: py-lock From 16bb89d62a4982f061f9a044e40456956846e4ca Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 11:46:57 -0700 Subject: [PATCH 018/128] initial --- app/dao/services_dao.py | 20 ++++++++++++++------ app/service_invite/rest.py | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 19755edfe..9ac63327f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -51,14 +51,22 @@ from app.utils import ( def dao_fetch_all_services(only_active=False): - query = Service.query.order_by(asc(Service.created_at)).options( - joinedload(Service.users) + + stmt = ( + select(Service) + .order_by(asc(Service.created_at)) + .options(joinedload(Service.users)) ) - if only_active: - query = query.filter(Service.active) - - return query.all() + stmt = ( + select(Service) + .where(Service.active is True) + .order_by(asc(Service.created_at)) + .options(joinedload(Service.users)) + ) + if only_active: + stmt = stmt.filter(Service.active) + return db.session.execute(stmt).scalars().all() def get_services_by_partial_name(service_name): diff --git a/app/service_invite/rest.py b/app/service_invite/rest.py index f6d9627da..5728b3ed5 100644 --- a/app/service_invite/rest.py +++ b/app/service_invite/rest.py @@ -86,7 +86,7 @@ def _create_service_invite(invited_user, invite_link_host): redis_store.set( f"email-personalisation-{saved_notification.id}", json.dumps(personalisation), - ex=2*24*60*60, + ex=2 * 24 * 60 * 60, ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) From bf271bfa222e55ec5a6314e528601202ec65c5d0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 11:59:06 -0700 Subject: [PATCH 019/128] initial --- app/dao/services_dao.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 9ac63327f..39c9dcbef 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -64,9 +64,8 @@ def dao_fetch_all_services(only_active=False): .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - if only_active: - stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalars().all() + result = db.session.execute(stmt) + return result.scalars().all() def get_services_by_partial_name(service_name): From bd334ffb42b803135d1ea6f1a87b511fd46c3393 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 12:11:02 -0700 Subject: [PATCH 020/128] initial --- app/dao/services_dao.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 39c9dcbef..5099ef439 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -3,7 +3,7 @@ from datetime import timedelta from flask import current_app from sqlalchemy import Float, cast, select -from sqlalchemy.orm import joinedload +from sqlalchemy.orm import joinedload, Session from sqlalchemy.sql.expression import and_, asc, case, func from app import db @@ -51,21 +51,21 @@ from app.utils import ( def dao_fetch_all_services(only_active=False): - - stmt = ( - select(Service) - .order_by(asc(Service.created_at)) - .options(joinedload(Service.users)) - ) - if only_active: + with Session() as session: stmt = ( select(Service) - .where(Service.active is True) .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - result = db.session.execute(stmt) - return result.scalars().all() + if only_active: + stmt = ( + select(Service) + .where(Service.active is True) + .order_by(asc(Service.created_at)) + .options(joinedload(Service.users)) + ) + result = db.session.execute(stmt) + return result.scalars().all() def get_services_by_partial_name(service_name): From 115233dec72a61c1a3d73fd6d53372cc263f4c3a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 12:15:51 -0700 Subject: [PATCH 021/128] initial --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 5099ef439..66c8ca58e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -64,7 +64,7 @@ def dao_fetch_all_services(only_active=False): .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - result = db.session.execute(stmt) + result = session.execute(stmt) return result.scalars().all() From 149ee70547c3ec77ed3b7036010205acb796b77d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 12:19:29 -0700 Subject: [PATCH 022/128] initial --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 66c8ca58e..0f87c1410 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -3,7 +3,7 @@ from datetime import timedelta from flask import current_app from sqlalchemy import Float, cast, select -from sqlalchemy.orm import joinedload, Session +from sqlalchemy.orm import Session, joinedload from sqlalchemy.sql.expression import and_, asc, case, func from app import db From 38194873285d79ab6e79f6fbbded29c1a90b6cb6 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 12:53:12 -0700 Subject: [PATCH 023/128] initial --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 0f87c1410..38ced647f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -51,7 +51,7 @@ from app.utils import ( def dao_fetch_all_services(only_active=False): - with Session() as session: + with Session(db.engine) as session: stmt = ( select(Service) .order_by(asc(Service.created_at)) From c1a179976426a6d5ff9d9947ff272f1bb49b9149 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 13:04:29 -0700 Subject: [PATCH 024/128] initial --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 38ced647f..281529d7c 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -64,7 +64,7 @@ def dao_fetch_all_services(only_active=False): .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - result = session.execute(stmt) + result = session.execute(stmt).unique() return result.scalars().all() From 49fd034b34fa916112e100f4ad37eb57db06408d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 13:15:10 -0700 Subject: [PATCH 025/128] try by not closing session --- app/dao/services_dao.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 281529d7c..ff4d372de 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -3,7 +3,7 @@ from datetime import timedelta from flask import current_app from sqlalchemy import Float, cast, select -from sqlalchemy.orm import Session, joinedload +from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import and_, asc, case, func from app import db @@ -51,21 +51,21 @@ from app.utils import ( def dao_fetch_all_services(only_active=False): - with Session(db.engine) as session: + + stmt = ( + select(Service) + .order_by(asc(Service.created_at)) + .options(joinedload(Service.users)) + ) + if only_active: stmt = ( select(Service) + .where(Service.active is True) .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - if only_active: - stmt = ( - select(Service) - .where(Service.active is True) - .order_by(asc(Service.created_at)) - .options(joinedload(Service.users)) - ) - result = session.execute(stmt).unique() - return result.scalars().all() + result = db.session.execute(stmt).unique() + return result.scalars().all() def get_services_by_partial_name(service_name): From b94b2b7b84a1cd671ea462c55ba533783bbc03b7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 13:29:42 -0700 Subject: [PATCH 026/128] try by not closing session --- app/dao/services_dao.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index ff4d372de..2e05af00f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -52,20 +52,15 @@ from app.utils import ( def dao_fetch_all_services(only_active=False): - stmt = ( - select(Service) - .order_by(asc(Service.created_at)) - .options(joinedload(Service.users)) - ) + stmt = select(Service) + if only_active: - stmt = ( - select(Service) - .where(Service.active is True) - .order_by(asc(Service.created_at)) - .options(joinedload(Service.users)) - ) - result = db.session.execute(stmt).unique() - return result.scalars().all() + stmt = stmt.where(Service.active) + + stmt = stmt.order_by(asc(Service.created_at)).options(joinedload(Service.users)) + + result = db.session.execute(stmt) + return result.unique().scalars().one() def get_services_by_partial_name(service_name): From cc3a6235410914bcc4d8d95d0a080ba694619164 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 13:37:20 -0700 Subject: [PATCH 027/128] try by not closing session --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 2e05af00f..d442e8305 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -60,7 +60,7 @@ def dao_fetch_all_services(only_active=False): stmt = stmt.order_by(asc(Service.created_at)).options(joinedload(Service.users)) result = db.session.execute(stmt) - return result.unique().scalars().one() + return result.unique().scalars().all() def get_services_by_partial_name(service_name): From 8f434d00042fc047b9fe0d610b7aabc09f4690b0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 13:50:12 -0700 Subject: [PATCH 028/128] try by not closing session --- app/dao/services_dao.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index d442e8305..28bf48881 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -65,7 +65,9 @@ def dao_fetch_all_services(only_active=False): def get_services_by_partial_name(service_name): service_name = escape_special_characters(service_name) - return Service.query.filter(Service.name.ilike("%{}%".format(service_name))).all() + stmt = select(Service).where(Service.name.ilike("%{}%".format(service_name))) + result = db.session.execute(stmt) + return result.scalars.all() def dao_count_live_services(): From 964464ed3fad2a6ce86954fdee6f4ac39a55e862 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 14:22:00 -0700 Subject: [PATCH 029/128] fix another --- app/dao/services_dao.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 28bf48881..1d558be1f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -67,7 +67,7 @@ def get_services_by_partial_name(service_name): service_name = escape_special_characters(service_name) stmt = select(Service).where(Service.name.ilike("%{}%".format(service_name))) result = db.session.execute(stmt) - return result.scalars.all() + return result.scalars().all() def dao_count_live_services(): @@ -191,14 +191,18 @@ def dao_fetch_service_by_id(service_id, only_active=False): def dao_fetch_service_by_inbound_number(number): - inbound_number = InboundNumber.query.filter( + stmt = select(InboundNumber).where( InboundNumber.number == number, InboundNumber.active - ).first() + ) + result = db.session.execute(stmt) + inbound_number = result.scalars().first() if not inbound_number: return None - return Service.query.filter(Service.id == inbound_number.service_id).first() + stmt = select(Service).where(Service.id == inbound_number.service_id) + result = db.session.execute(stmt) + return result.scalars().first() def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): From 213b36e4de3bae2d53d2e9566f10e1aacde1c4fa Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 14:34:12 -0700 Subject: [PATCH 030/128] fix another --- app/dao/services_dao.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 1d558be1f..22c0e8395 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -71,11 +71,11 @@ def get_services_by_partial_name(service_name): def dao_count_live_services(): - return Service.query.filter_by( - active=True, - restricted=False, - count_as_live=True, - ).count() + stmt = select(Service).where( + Service.active, Service.count_as_live, Service.restricted is False + ) + result = db.session.execute(stmt) + return result.scalars().count() def dao_fetch_live_services_data(): @@ -228,11 +228,11 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): def dao_fetch_all_services_created_by_user(user_id): - query = Service.query.filter_by(created_by_id=user_id).order_by( - asc(Service.created_at) + stmt = ( + select(Service).where(created_by_id=user_id).order_by(asc(Service.created_at)) ) - - return query.all() + result = db.session.execute(stmt) + return result.scalars.all() @autocommit From 60148a2848c4aa2b3ea54e2938063176225dbb31 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 14:43:56 -0700 Subject: [PATCH 031/128] fix another --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 22c0e8395..b66cafa59 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -75,7 +75,7 @@ def dao_count_live_services(): Service.active, Service.count_as_live, Service.restricted is False ) result = db.session.execute(stmt) - return result.scalars().count() + return result.count() def dao_fetch_live_services_data(): From a41fa318a8442d151e89512ffcfa462f3944d803 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 14:52:05 -0700 Subject: [PATCH 032/128] fix another --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index b66cafa59..96efe5041 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -75,7 +75,7 @@ def dao_count_live_services(): Service.active, Service.count_as_live, Service.restricted is False ) result = db.session.execute(stmt) - return result.count() + return result.scalar() def dao_fetch_live_services_data(): From df93bbb45b5446736c122b697a997b5b060ecd53 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 15:01:46 -0700 Subject: [PATCH 033/128] fix another --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 96efe5041..93a8692df 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -75,7 +75,7 @@ def dao_count_live_services(): Service.active, Service.count_as_live, Service.restricted is False ) result = db.session.execute(stmt) - return result.scalar() + return result.scalars() def dao_fetch_live_services_data(): From aab06dc5ab2256fc23b95893727adf5ef6487d60 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 15:12:01 -0700 Subject: [PATCH 034/128] fix another --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 93a8692df..0dfd1c36b 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -72,10 +72,10 @@ def get_services_by_partial_name(service_name): def dao_count_live_services(): stmt = select(Service).where( - Service.active, Service.count_as_live, Service.restricted is False + Service.active, Service.count_as_live, Service.restricted == False ) result = db.session.execute(stmt) - return result.scalars() + return result.scalar() # Retrieves the count def dao_fetch_live_services_data(): From 6fc889db04fe5acbf8f2eb47df30efe7628436cd Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 15:16:04 -0700 Subject: [PATCH 035/128] fix another --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 0dfd1c36b..9fdae8636 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -72,10 +72,10 @@ def get_services_by_partial_name(service_name): def dao_count_live_services(): stmt = select(Service).where( - Service.active, Service.count_as_live, Service.restricted == False + Service.active, Service.count_as_live, Service.restricted == False # noqa ) result = db.session.execute(stmt) - return result.scalar() # Retrieves the count + return result.scalar() # Retrieves the count def dao_fetch_live_services_data(): From 40ff981b7935d80eb3565314baaf6b236b12e8ce Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 8 Oct 2024 15:27:20 -0700 Subject: [PATCH 036/128] fix another --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 9fdae8636..139e7c9a4 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -71,7 +71,7 @@ def get_services_by_partial_name(service_name): def dao_count_live_services(): - stmt = select(Service).where( + stmt = select(func.count()).select_from(Service).where( Service.active, Service.count_as_live, Service.restricted == False # noqa ) result = db.session.execute(stmt) From d5979286863c951335e2c5d0478d3de87217415f Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 07:08:01 -0700 Subject: [PATCH 037/128] convert more queries --- app/dao/services_dao.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 139e7c9a4..c853f0596 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -71,8 +71,12 @@ def get_services_by_partial_name(service_name): def dao_count_live_services(): - stmt = select(func.count()).select_from(Service).where( - Service.active, Service.count_as_live, Service.restricted == False # noqa + stmt = ( + select(func.count()) + .select_from(Service) + .where( + Service.active, Service.count_as_live, Service.restricted == False # noqa + ) ) result = db.session.execute(stmt) return result.scalar() # Retrieves the count @@ -267,11 +271,19 @@ def dao_archive_service(service_id): def dao_fetch_service_by_id_and_user(service_id, user_id): - return ( - Service.query.filter(Service.users.any(id=user_id), Service.id == service_id) + # return ( + # Service.query.filter(Service.users.any(id=user_id), Service.id == service_id) + # .options(joinedload(Service.users)) + # .one() + # ) + + stmt = ( + select(Service.users.any(id=user_id), Service.id == service_id) + .select_from(Service) .options(joinedload(Service.users)) - .one() ) + result = db.session.execute(stmt) + return result.scalars().one() @autocommit @@ -565,14 +577,22 @@ def dao_suspend_service(service_id): @autocommit @version_class(Service) def dao_resume_service(service_id): - service = Service.query.get(service_id) + # service = Service.query.get(service_id) + stmt = select(Service).where(id == service_id) + result = db.session.execute(stmt) + service = result.scalars().one() + service.active = True def dao_fetch_active_users_for_service(service_id): - query = User.query.filter(User.services.any(id=service_id), User.state == "active") + # query = User.query.filter(User.services.any(id=service_id), User.state == "active") - return query.all() + # return query.all() + + stmt = select(User).where(User.services.any(id=service_id), User.state == "active") + result = db.session.execute(stmt) + return result.scalars().all() def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500): From c8a8290053679ad5142411ade71f515c1b588219 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 07:35:13 -0700 Subject: [PATCH 038/128] convert more queries --- app/dao/services_dao.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index c853f0596..d78d108f4 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -278,12 +278,11 @@ def dao_fetch_service_by_id_and_user(service_id, user_id): # ) stmt = ( - select(Service.users.any(id=user_id), Service.id == service_id) - .select_from(Service) + select(Service).filter(Service.users.any(id=user_id), Service.id == service_id) .options(joinedload(Service.users)) ) - result = db.session.execute(stmt) - return result.scalars().one() + result = db.session.execute(stmt).scalar_one() + return result @autocommit From bf822bf74f286ccbb3a4b8335cbc87811a653ca0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 08:00:17 -0700 Subject: [PATCH 039/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index d78d108f4..658eac96d 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -579,7 +579,7 @@ def dao_resume_service(service_id): # service = Service.query.get(service_id) stmt = select(Service).where(id == service_id) result = db.session.execute(stmt) - service = result.scalars().one() + service = result.scalar_one() service.active = True From d1fb503f37ee8dc5e240d0a697ec201b82df3766 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 08:10:10 -0700 Subject: [PATCH 040/128] convert more queries --- app/dao/services_dao.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 658eac96d..42cfa7227 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -577,9 +577,7 @@ def dao_suspend_service(service_id): @version_class(Service) def dao_resume_service(service_id): # service = Service.query.get(service_id) - stmt = select(Service).where(id == service_id) - result = db.session.execute(stmt) - service = result.scalar_one() + service = db.session.get(Service, service_id) service.active = True From dc4902835eb3e9b9004b80726f28326cbe7d5ec5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 08:34:40 -0700 Subject: [PATCH 041/128] convert more queries --- app/dao/services_dao.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 42cfa7227..e86fab52a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -232,8 +232,15 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): def dao_fetch_all_services_created_by_user(user_id): + + # query = Service.query.filter_by(created_by_id=user_id).order_by(asc(Service.created_at) + + # return query.all() + stmt = ( - select(Service).where(created_by_id=user_id).order_by(asc(Service.created_at)) + select(Service) + .filter_by(created_by_id=user_id) + .order_by(asc(Service.created_at)) ) result = db.session.execute(stmt) return result.scalars.all() @@ -278,7 +285,8 @@ def dao_fetch_service_by_id_and_user(service_id, user_id): # ) stmt = ( - select(Service).filter(Service.users.any(id=user_id), Service.id == service_id) + select(Service) + .filter(Service.users.any(id=user_id), Service.id == service_id) .options(joinedload(Service.users)) ) result = db.session.execute(stmt).scalar_one() From c9b5bf5d0bae2bb5620af641ba51f66827e5ada9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 09:05:14 -0700 Subject: [PATCH 042/128] convert more queries --- app/dao/services_dao.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e86fab52a..c688caecd 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -233,17 +233,19 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): def dao_fetch_all_services_created_by_user(user_id): - # query = Service.query.filter_by(created_by_id=user_id).order_by(asc(Service.created_at) - - # return query.all() - - stmt = ( - select(Service) - .filter_by(created_by_id=user_id) - .order_by(asc(Service.created_at)) + query = Service.query.filter_by(created_by_id=user_id).order_by( + asc(Service.created_at) ) - result = db.session.execute(stmt) - return result.scalars.all() + + return query.all() + + # stmt = ( + # select(Service) + # .filter_by(created_by_id=user_id) + # .order_by(asc(Service.created_at)) + # ) + # result = db.session.execute(stmt) + # return result.scalars.all() @autocommit From 6af03ff8aa0c950a5e737284c0ca9bf1d50ca25b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 11:56:02 -0700 Subject: [PATCH 043/128] convert more queries --- app/dao/services_dao.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index c688caecd..e3124c8ad 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -233,19 +233,19 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): def dao_fetch_all_services_created_by_user(user_id): - query = Service.query.filter_by(created_by_id=user_id).order_by( - asc(Service.created_at) + # query = Service.query.filter_by(created_by_id=user_id).order_by( + # asc(Service.created_at) + # ) + + # return query.all() + + stmt = ( + select(Service) + .filter_by(created_by_id=user_id) + .order_by(asc(Service.created_at)) ) - return query.all() - - # stmt = ( - # select(Service) - # .filter_by(created_by_id=user_id) - # .order_by(asc(Service.created_at)) - # ) - # result = db.session.execute(stmt) - # return result.scalars.all() + return db.session.scalars(stmt).all() @autocommit From 29eb9627e9ebabb5d0c8d721ca178b24a4cc5e0d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 12:15:01 -0700 Subject: [PATCH 044/128] convert more queries --- app/dao/services_dao.py | 53 +++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e3124c8ad..60eed6b70 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -210,25 +210,41 @@ def dao_fetch_service_by_inbound_number(number): def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): - query = Service.query.filter_by(id=service_id).options(joinedload(Service.api_keys)) + # query = Service.query.filter_by(id=service_id).options(joinedload(Service.api_keys)) + # if only_active: + # query = query.filter(Service.active) + + # return query.one() + stmt = ( + select(Service).filter_by(id=service_id).options(joinedload(Service.api_keys)) + ) if only_active: - query = query.filter(Service.active) - - return query.one() + stmt = stmt.filter(Service.working) + return db.session.scalar(stmt.one()) def dao_fetch_all_services_by_user(user_id, only_active=False): - query = ( - Service.query.filter(Service.users.any(id=user_id)) + # query = ( + # Service.query.filter(Service.users.any(id=user_id)) + # .order_by(asc(Service.created_at)) + # .options(joinedload(Service.users)) + # ) + + # if only_active: + # query = query.filter(Service.active) + + # return query.all() + + stmt = ( + select(Service) + .filter(Service.users.any(id=user_id)) .order_by(asc(Service.created_at)) .options(joinedload(Service.users)) ) - if only_active: - query = query.filter(Service.active) - - return query.all() + stmt = stmt.filter(Service.active) + return db.session.scalar(stmt.one()) def dao_fetch_all_services_created_by_user(user_id): @@ -257,14 +273,21 @@ def dao_fetch_all_services_created_by_user(user_id): def dao_archive_service(service_id): # have to eager load templates and api keys so that we don't flush when we loop through them # to ensure that db.session still contains the models when it comes to creating history objects - service = ( - Service.query.options( + # service = ( + # Service.query.options( + # joinedload(Service.templates).subqueryload(Template.template_redacted), + # joinedload(Service.api_keys), + # ) + # .filter(Service.id == service_id) + # .one() + # ) + stmt = select( + Service.options( joinedload(Service.templates).subqueryload(Template.template_redacted), joinedload(Service.api_keys), ) - .filter(Service.id == service_id) - .one() - ) + ).filter(Service.id == service_id) + service = db.session.scalars(stmt.one()) service.active = False service.name = get_archived_db_column_value(service.name) From ba787b0febd850015057e38e9b5575ab683513c0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 12:40:42 -0700 Subject: [PATCH 045/128] convert more queries --- app/dao/services_dao.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 60eed6b70..5277e09ef 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -282,11 +282,10 @@ def dao_archive_service(service_id): # .one() # ) stmt = select( - Service.options( + Service).options( joinedload(Service.templates).subqueryload(Template.template_redacted), joinedload(Service.api_keys), - ) - ).filter(Service.id == service_id) + ).filter(Service.id == service_id) service = db.session.scalars(stmt.one()) service.active = False From 18053205ece6c81c3235138629820d175725f459 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 12:51:21 -0700 Subject: [PATCH 046/128] convert more queries --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 5277e09ef..82c6874d4 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -261,7 +261,7 @@ def dao_fetch_all_services_created_by_user(user_id): .order_by(asc(Service.created_at)) ) - return db.session.scalars(stmt).all() + return db.session.execute(stmt).scalars().all() @autocommit @@ -286,7 +286,7 @@ def dao_archive_service(service_id): joinedload(Service.templates).subqueryload(Template.template_redacted), joinedload(Service.api_keys), ).filter(Service.id == service_id) - service = db.session.scalars(stmt.one()) + service = db.session.execute(stmt).scalars().one() service.active = False service.name = get_archived_db_column_value(service.name) From 0f5453165c7aa99ef8a30797526bdc80a215a24d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:08:47 -0700 Subject: [PATCH 047/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 82c6874d4..ebf12469a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -286,7 +286,7 @@ def dao_archive_service(service_id): joinedload(Service.templates).subqueryload(Template.template_redacted), joinedload(Service.api_keys), ).filter(Service.id == service_id) - service = db.session.execute(stmt).scalars().one() + service = db.session.execute(stmt).scalars().unique().one() service.active = False service.name = get_archived_db_column_value(service.name) From f02a0b247929da3c0a9d08eaff7be3cdf8c35730 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:18:41 -0700 Subject: [PATCH 048/128] convert more queries --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index ebf12469a..05a75685a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -221,7 +221,7 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.working) - return db.session.scalar(stmt.one()) + return db.session.execute(stmt).scalar().one() def dao_fetch_all_services_by_user(user_id, only_active=False): @@ -244,7 +244,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.active) - return db.session.scalar(stmt.one()) + return db.session.execute(stmt).scalar().one() def dao_fetch_all_services_created_by_user(user_id): From 298f589833ece93509fe67aa90c51bfa624b72f1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:28:31 -0700 Subject: [PATCH 049/128] convert more queries --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 05a75685a..2c8c36dad 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -221,7 +221,7 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.working) - return db.session.execute(stmt).scalar().one() + return db.session.execute(stmt).scalars().one() def dao_fetch_all_services_by_user(user_id, only_active=False): @@ -244,7 +244,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalar().one() + return db.session.execute(stmt).scalars().one() def dao_fetch_all_services_created_by_user(user_id): From 3cc398023401d21a3c5a7297851b6db921fecdfc Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:38:14 -0700 Subject: [PATCH 050/128] convert more queries --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 2c8c36dad..fe19ee7e1 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -191,7 +191,7 @@ def dao_fetch_service_by_id(service_id, only_active=False): stmt = stmt.where(Service.active) result = db.session.execute(stmt) - return result.unique().scalars().one() + return result.unique().scalars().unique().one() def dao_fetch_service_by_inbound_number(number): @@ -244,7 +244,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalars().one() + return db.session.execute(stmt).scalars().unique().one() def dao_fetch_all_services_created_by_user(user_id): From bba76a3dfce2ef9db8f9d30f0dfda8ebfe9c3213 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:48:06 -0700 Subject: [PATCH 051/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index fe19ee7e1..5a3771867 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -244,7 +244,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalars().unique().one() + return db.session.execute(stmt).scalars().all() def dao_fetch_all_services_created_by_user(user_id): From 61114e86dd32fb54383ef7b7428c79b5714b13f0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 13:56:26 -0700 Subject: [PATCH 052/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 5a3771867..e2594f337 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -244,7 +244,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalars().all() + return db.session.execute(stmt).scalars().unique().all() def dao_fetch_all_services_created_by_user(user_id): From 2c870d20f46a1b3bf7f42bc6d9176cc33a888258 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 14:05:11 -0700 Subject: [PATCH 053/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e2594f337..687145fe2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -221,7 +221,7 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): ) if only_active: stmt = stmt.filter(Service.working) - return db.session.execute(stmt).scalars().one() + return db.session.execute(stmt).scalars().unique().one() def dao_fetch_all_services_by_user(user_id, only_active=False): From f771c30430e3ee0650e659c0144ee9c9f1ed4c59 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 14:12:51 -0700 Subject: [PATCH 054/128] convert more queries --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 687145fe2..3fcdbf3ef 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -220,7 +220,7 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): select(Service).filter_by(id=service_id).options(joinedload(Service.api_keys)) ) if only_active: - stmt = stmt.filter(Service.working) + stmt = stmt.filter(Service.active) return db.session.execute(stmt).scalars().unique().one() From ae60cbe5641a79c482c64fb4be222abc9616999e Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 14:23:43 -0700 Subject: [PATCH 055/128] convert more queries --- app/dao/services_dao.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 3fcdbf3ef..e5725aedf 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -704,23 +704,34 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 def get_live_services_with_organization(): - query = ( - db.session.query( - Service.id.label("service_id"), + # query = ( + # db.session.query( + # Service.id.label("service_id"), + # Service.name.label("service_name"), + # Organization.id.label("organization_id"), + # Organization.name.label("organization_name"), + # ) + # .outerjoin(Service.organization) + # .filter( + # Service.count_as_live.is_(True), + # Service.active.is_(True), + # Service.restricted.is_(False), + # ) + # .order_by(Organization.name, Service.name) + # ) + + # return query.all() + + stmt = select(Service.id.label("service_id"), Service.name.label("service_name"), Organization.id.label("organization_id"), - Organization.name.label("organization_name"), - ) - .outerjoin(Service.organization) - .filter( + Organization.name.label("organization_name")).select_from(Service).outerjoin(Service.organization).filter( Service.count_as_live.is_(True), Service.active.is_(True), Service.restricted.is_(False), - ) - .order_by(Organization.name, Service.name) - ) + ).order_by(Organization.name, Service.name) - return query.all() + return db.session.execute(stmt).all() def fetch_notification_stats_for_service_by_month_by_user( From 85b88c046ed9c976f329e6257ac0ffb0fd54ff00 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 14:34:11 -0700 Subject: [PATCH 056/128] convert more queries --- app/dao/services_dao.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e5725aedf..581a950d9 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -281,11 +281,14 @@ def dao_archive_service(service_id): # .filter(Service.id == service_id) # .one() # ) - stmt = select( - Service).options( + stmt = ( + select(Service) + .options( joinedload(Service.templates).subqueryload(Template.template_redacted), joinedload(Service.api_keys), - ).filter(Service.id == service_id) + ) + .filter(Service.id == service_id) + ) service = db.session.execute(stmt).scalars().unique().one() service.active = False @@ -722,14 +725,22 @@ def get_live_services_with_organization(): # return query.all() - stmt = select(Service.id.label("service_id"), + stmt = ( + select( + Service.id.label("service_id"), Service.name.label("service_name"), Organization.id.label("organization_id"), - Organization.name.label("organization_name")).select_from(Service).outerjoin(Service.organization).filter( + Organization.name.label("organization_name"), + ) + .select_from(Service) + .outerjoin(Service.organization) + .filter( Service.count_as_live.is_(True), Service.active.is_(True), Service.restricted.is_(False), - ).order_by(Organization.name, Service.name) + ) + .order_by(Organization.name, Service.name) + ) return db.session.execute(stmt).all() From c1a4c7e508c2ca73169cf4a5997917d1080a2914 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 9 Oct 2024 14:48:07 -0700 Subject: [PATCH 057/128] convert more queries --- app/dao/services_dao.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 581a950d9..9bbaa5c75 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -514,13 +514,35 @@ def dao_fetch_stats_for_service_from_days_for_user( start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - return ( - db.session.query( + # return ( + # db.session.query( + # NotificationAllTimeView.notification_type, + # NotificationAllTimeView.status, + # func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + # func.count(NotificationAllTimeView.id).label("count"), + # ) + # .filter( + # NotificationAllTimeView.service_id == service_id, + # NotificationAllTimeView.key_type != KeyType.TEST, + # NotificationAllTimeView.created_at >= start_date, + # NotificationAllTimeView.created_at < end_date, + # NotificationAllTimeView.created_by_id == user_id, + # ) + # .group_by( + # NotificationAllTimeView.notification_type, + # NotificationAllTimeView.status, + # func.date_trunc("day", NotificationAllTimeView.created_at), + # ) + # .all() + # ) + stmt = ( + select( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) + .select_from(NotificationAllTimeView) .filter( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, @@ -533,8 +555,8 @@ def dao_fetch_stats_for_service_from_days_for_user( NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at), ) - .all() ) + return db.session.execute(stmt).scalars().all() def dao_fetch_todays_stats_for_all_services( From cbd2cd132fd105d16c710968d218e02cfed1d4be Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 07:22:15 -0700 Subject: [PATCH 058/128] convert service_permissions_dao --- app/dao/service_permissions_dao.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/app/dao/service_permissions_dao.py b/app/dao/service_permissions_dao.py index e459b6e56..95a40c903 100644 --- a/app/dao/service_permissions_dao.py +++ b/app/dao/service_permissions_dao.py @@ -1,12 +1,17 @@ +from sqlalchemy import delete, select + from app import db from app.dao.dao_utils import autocommit from app.models import ServicePermission def dao_fetch_service_permissions(service_id): - return ServicePermission.query.filter( - ServicePermission.service_id == service_id - ).all() + # return ServicePermission.query.filter( + # ServicePermission.service_id == service_id + # ).all() + + stmt = select(ServicePermission).filter(ServicePermission.service_id == service_id) + return db.session.execute(stmt).scalars().all() @autocommit @@ -16,9 +21,17 @@ def dao_add_service_permission(service_id, permission): def dao_remove_service_permission(service_id, permission): - deleted = ServicePermission.query.filter( + # deleted = ServicePermission.query.filter( + # ServicePermission.service_id == service_id, + # ServicePermission.permission == permission, + # ).delete() + # db.session.commit() + # return deleted + + stmt = delete(ServicePermission).where( ServicePermission.service_id == service_id, ServicePermission.permission == permission, - ).delete() + ) + result = db.session.execute(stmt) db.session.commit() - return deleted + return result.rowcount From 2db45c8b508b6312f3a946340613589302c8a592 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 07:49:26 -0700 Subject: [PATCH 059/128] convert other daos --- app/dao/service_sms_sender_dao.py | 22 ++++++++++++++++------ app/dao/service_user_dao.py | 25 ++++++++++++++++++------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/app/dao/service_sms_sender_dao.py b/app/dao/service_sms_sender_dao.py index 9224cf09d..c23bd5a2e 100644 --- a/app/dao/service_sms_sender_dao.py +++ b/app/dao/service_sms_sender_dao.py @@ -1,4 +1,4 @@ -from sqlalchemy import desc +from sqlalchemy import desc, select from app import db from app.dao.dao_utils import autocommit @@ -17,17 +17,27 @@ def insert_service_sms_sender(service, sms_sender): def dao_get_service_sms_senders_by_id(service_id, service_sms_sender_id): - return ServiceSmsSender.query.filter_by( + # return ServiceSmsSender.query.filter_by( + # id=service_sms_sender_id, service_id=service_id, archived=False + # ).one() + stmt = select(ServiceSmsSender).filter_by( id=service_sms_sender_id, service_id=service_id, archived=False - ).one() + ) + return db.session.execute(stmt).scalars().one() def dao_get_sms_senders_by_service_id(service_id): - return ( - ServiceSmsSender.query.filter_by(service_id=service_id, archived=False) + # return ( + # ServiceSmsSender.query.filter_by(service_id=service_id, archived=False) + # .order_by(desc(ServiceSmsSender.is_default)) + # .all() + # ) + stmt = ( + select(ServiceSmsSender) + .filter_by(ervice_id=service_id, archived=False) .order_by(desc(ServiceSmsSender.is_default)) - .all() ) + return db.session.execute(stmt).scalars().all() @autocommit diff --git a/app/dao/service_user_dao.py b/app/dao/service_user_dao.py index 0b991a4fc..b02005a3f 100644 --- a/app/dao/service_user_dao.py +++ b/app/dao/service_user_dao.py @@ -1,3 +1,5 @@ +from sqlalchemy import select + from app import db from app.dao.dao_utils import autocommit from app.models import ServiceUser, User @@ -7,19 +9,28 @@ def dao_get_service_user(user_id, service_id): # TODO: This has been changed to account for the test case failure # that used this method but have any service user to return. Somehow, this # started to throw an error with one() method in sqlalchemy 2.0 unlike 1.4 - return ServiceUser.query.filter_by( - user_id=user_id, service_id=service_id - ).one_or_none() + # return ServiceUser.query.filter_by( + # user_id=user_id, service_id=service_id + # ).one_or_none() + stmt = select(ServiceUser).filter_by(user_id=user_id, service_id=service_id) + return db.session.execute(stmt).scalars().one_or_none() def dao_get_active_service_users(service_id): - query = ( - db.session.query(ServiceUser) + # query = ( + # db.session.query(ServiceUser) + # .join(User, User.id == ServiceUser.user_id) + # .filter(User.state == "active", ServiceUser.service_id == service_id) + # ) + + # return query.all() + + stmt = ( + select(ServiceUser) .join(User, User.id == ServiceUser.user_id) .filter(User.state == "active", ServiceUser.service_id == service_id) ) - - return query.all() + return db.session.execute(stmt).scalars().all() def dao_get_service_users_by_user_id(user_id): From 2cd7cc4665d4298cf884765eacf89cb1894f52e4 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 08:10:19 -0700 Subject: [PATCH 060/128] convert other daos --- app/dao/service_sms_sender_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/service_sms_sender_dao.py b/app/dao/service_sms_sender_dao.py index c23bd5a2e..df0f2a3e9 100644 --- a/app/dao/service_sms_sender_dao.py +++ b/app/dao/service_sms_sender_dao.py @@ -34,7 +34,7 @@ def dao_get_sms_senders_by_service_id(service_id): # ) stmt = ( select(ServiceSmsSender) - .filter_by(ervice_id=service_id, archived=False) + .filter_by(service_id=service_id, archived=False) .order_by(desc(ServiceSmsSender.is_default)) ) return db.session.execute(stmt).scalars().all() From 36a834697269030f452c8e119f566ecd98a5460c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 08:25:34 -0700 Subject: [PATCH 061/128] convert other daos --- app/dao/services_dao.py | 96 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 5 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 9bbaa5c75..d65d57ddc 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -85,8 +85,90 @@ def dao_count_live_services(): def dao_fetch_live_services_data(): year_start_date, year_end_date = get_current_calendar_year() + # most_recent_annual_billing = ( + # db.session.query( + # AnnualBilling.service_id, + # func.max(AnnualBilling.financial_year_start).label("year"), + # ) + # .group_by(AnnualBilling.service_id) + # .subquery() + # ) + + # this_year_ft_billing = FactBilling.query.filter( + # FactBilling.local_date >= year_start_date, + # FactBilling.local_date <= year_end_date, + # ).subquery() + + # data = ( + # db.session.query( + # Service.id.label("service_id"), + # Service.name.label("service_name"), + # Organization.name.label("organization_name"), + # Organization.organization_type.label("organization_type"), + # Service.consent_to_research.label("consent_to_research"), + # User.name.label("contact_name"), + # User.email_address.label("contact_email"), + # User.mobile_number.label("contact_mobile"), + # Service.go_live_at.label("live_date"), + # Service.volume_sms.label("sms_volume_intent"), + # Service.volume_email.label("email_volume_intent"), + # case( + # ( + # this_year_ft_billing.c.notification_type == NotificationType.EMAIL, + # func.sum(this_year_ft_billing.c.notifications_sent), + # ), + # else_=0, + # ).label("email_totals"), + # case( + # ( + # this_year_ft_billing.c.notification_type == NotificationType.SMS, + # func.sum(this_year_ft_billing.c.notifications_sent), + # ), + # else_=0, + # ).label("sms_totals"), + # AnnualBilling.free_sms_fragment_limit, + # ) + # .join(Service.annual_billing) + # .join( + # most_recent_annual_billing, + # and_( + # Service.id == most_recent_annual_billing.c.service_id, + # AnnualBilling.financial_year_start == most_recent_annual_billing.c.year, + # ), + # ) + # .outerjoin(Service.organization) + # .outerjoin( + # this_year_ft_billing, Service.id == this_year_ft_billing.c.service_id + # ) + # .outerjoin(User, Service.go_live_user_id == User.id) + # .filter( + # Service.count_as_live.is_(True), + # Service.active.is_(True), + # Service.restricted.is_(False), + # ) + # .group_by( + # Service.id, + # Organization.name, + # Organization.organization_type, + # Service.name, + # Service.consent_to_research, + # Service.count_as_live, + # Service.go_live_user_id, + # User.name, + # User.email_address, + # User.mobile_number, + # Service.go_live_at, + # Service.volume_sms, + # Service.volume_email, + # this_year_ft_billing.c.notification_type, + # AnnualBilling.free_sms_fragment_limit, + # ) + # .order_by(asc(Service.go_live_at)) + # .all() + # ) + most_recent_annual_billing = ( - db.session.query( + select( AnnualBilling.service_id, func.max(AnnualBilling.financial_year_start).label("year"), ) @@ -94,13 +176,13 @@ def dao_fetch_live_services_data(): .subquery() ) - this_year_ft_billing = FactBilling.query.filter( + this_year_ft_billing = select(FactBilling).filter( FactBilling.local_date >= year_start_date, FactBilling.local_date <= year_end_date, ).subquery() - data = ( - db.session.query( + stmt = ( + select( Service.id.label("service_id"), Service.name.label("service_name"), Organization.name.label("organization_name"), @@ -164,8 +246,12 @@ def dao_fetch_live_services_data(): AnnualBilling.free_sms_fragment_limit, ) .order_by(asc(Service.go_live_at)) - .all() ) + + data = db.session.execute(stmt).scalars().all() + + + results = [] for row in data: existing_service = next( From 556158012d2b8eda1b93798fc99ef01a42f33d84 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 08:33:14 -0700 Subject: [PATCH 062/128] convert other daos --- app/dao/services_dao.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index d65d57ddc..da237fcb2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -176,10 +176,14 @@ def dao_fetch_live_services_data(): .subquery() ) - this_year_ft_billing = select(FactBilling).filter( - FactBilling.local_date >= year_start_date, - FactBilling.local_date <= year_end_date, - ).subquery() + this_year_ft_billing = ( + select(FactBilling) + .filter( + FactBilling.local_date >= year_start_date, + FactBilling.local_date <= year_end_date, + ) + .subquery() + ) stmt = ( select( @@ -250,8 +254,6 @@ def dao_fetch_live_services_data(): data = db.session.execute(stmt).scalars().all() - - results = [] for row in data: existing_service = next( From 20585ce29ca729003f40a8e9cdcbb5f49ff17127 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 08:43:13 -0700 Subject: [PATCH 063/128] convert other daos --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index da237fcb2..b745985f3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -253,7 +253,7 @@ def dao_fetch_live_services_data(): ) data = db.session.execute(stmt).scalars().all() - + print(f"DATA IS {data}") results = [] for row in data: existing_service = next( From d845e0602b8534b2b418742b9d6857acb964dacd Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 08:55:17 -0700 Subject: [PATCH 064/128] convert other daos --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index b745985f3..7e7f99481 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -252,7 +252,7 @@ def dao_fetch_live_services_data(): .order_by(asc(Service.go_live_at)) ) - data = db.session.execute(stmt).scalars().all() + data = db.session.execute(stmt).all() print(f"DATA IS {data}") results = [] for row in data: From 299926df1eb5d4a7b56e6bc59c07367ccc04f7a9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 10:09:24 -0700 Subject: [PATCH 065/128] convert other daos --- app/dao/services_dao.py | 121 ++++++++++++++++++++++++++++++++++------ 1 file changed, 103 insertions(+), 18 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7e7f99481..7bb6e9c2f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -253,7 +253,6 @@ def dao_fetch_live_services_data(): ) data = db.session.execute(stmt).all() - print(f"DATA IS {data}") results = [] for row in data: existing_service = next( @@ -551,8 +550,25 @@ def delete_service_and_all_associated_db_objects(service): def dao_fetch_todays_stats_for_service(service_id): today = utc_now().date() start_date = get_midnight_in_utc(today) - return ( - db.session.query( + # return ( + # db.session.query( + # Notification.notification_type, + # Notification.status, + # func.count(Notification.id).label("count"), + # ) + # .filter( + # Notification.service_id == service_id, + # Notification.key_type != KeyType.TEST, + # Notification.created_at >= start_date, + # ) + # .group_by( + # Notification.notification_type, + # Notification.status, + # ) + # .all() + # ) + stmt = ( + select( Notification.notification_type, Notification.status, func.count(Notification.id).label("count"), @@ -566,16 +582,37 @@ def dao_fetch_todays_stats_for_service(service_id): Notification.notification_type, Notification.status, ) - .all() ) + return db.session.execute(stmt).all() def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - return ( - db.session.query( + # return ( + # db.session.query( + # NotificationAllTimeView.notification_type, + # NotificationAllTimeView.status, + # func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + # func.count(NotificationAllTimeView.id).label("count"), + # ) + # .filter( + # NotificationAllTimeView.service_id == service_id, + # NotificationAllTimeView.key_type != KeyType.TEST, + # NotificationAllTimeView.created_at >= start_date, + # NotificationAllTimeView.created_at < end_date, + # ) + # .group_by( + # NotificationAllTimeView.notification_type, + # NotificationAllTimeView.status, + # func.date_trunc("day", NotificationAllTimeView.created_at), + # ) + # .all() + # ) + + stmt = ( + select( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), @@ -592,8 +629,8 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at), ) - .all() ) + return db.session.execute(stmt).all() def dao_fetch_stats_for_service_from_days_for_user( @@ -703,13 +740,19 @@ def dao_fetch_todays_stats_for_all_services( def dao_suspend_service(service_id): # have to eager load api keys so that we don't flush when we loop through them # to ensure that db.session still contains the models when it comes to creating history objects - service = ( - Service.query.options( - joinedload(Service.api_keys), - ) + # service = ( + # Service.query.options( + # joinedload(Service.api_keys), + # ) + # .filter(Service.id == service_id) + # .one() + # ) + stmt = ( + select(Service) + .options(joinedload(Service.api_keys)) .filter(Service.id == service_id) - .one() ) + service = db.session.execute(stmt).one() for api_key in service.api_keys: if not api_key.expiry_date: @@ -738,8 +781,29 @@ def dao_fetch_active_users_for_service(service_id): def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500): - return ( - db.session.query( + # return ( + # db.session.query( + # Notification.service_id.label("service_id"), + # func.count(Notification.id).label("notification_count"), + # ) + # .filter( + # Notification.service_id == Service.id, + # Notification.created_at >= start_date, + # Notification.created_at <= end_date, + # Notification.key_type != KeyType.TEST, + # Notification.notification_type == NotificationType.SMS, + # func.substr(Notification.normalised_to, 3, 7) == "7700900", + # Service.restricted == False, # noqa + # Service.active == True, # noqa + # ) + # .group_by( + # Notification.service_id, + # ) + # .having(func.count(Notification.id) > threshold) + # .all() + # ) + stmt = ( + select( Notification.service_id.label("service_id"), func.count(Notification.id).label("notification_count"), ) @@ -757,8 +821,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500) Notification.service_id, ) .having(func.count(Notification.id) > threshold) - .all() ) + return db.session.execute(stmt).all() def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000): @@ -858,8 +922,29 @@ def get_live_services_with_organization(): def fetch_notification_stats_for_service_by_month_by_user( start_date, end_date, service_id, user_id ): - return ( - db.session.query( + # return ( + # db.session.query( + # func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + # NotificationAllTimeView.notification_type, + # (NotificationAllTimeView.status).label("notification_status"), + # func.count(NotificationAllTimeView.id).label("count"), + # ) + # .filter( + # NotificationAllTimeView.service_id == service_id, + # NotificationAllTimeView.created_at >= start_date, + # NotificationAllTimeView.created_at < end_date, + # NotificationAllTimeView.key_type != KeyType.TEST, + # NotificationAllTimeView.created_by_id == user_id, + # ) + # .group_by( + # func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + # NotificationAllTimeView.notification_type, + # NotificationAllTimeView.status, + # ) + # .all() + # ) + stmt = ( + select( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), NotificationAllTimeView.notification_type, (NotificationAllTimeView.status).label("notification_status"), @@ -877,8 +962,8 @@ def fetch_notification_stats_for_service_by_month_by_user( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, ) - .all() ) + return db.session.execute(stmt).all() def get_specific_days_stats(data, start_date, days=None, end_date=None): From ee65f4e718d682157f1397a4a144c6ff3efe7090 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 10:18:41 -0700 Subject: [PATCH 066/128] use unique() --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7bb6e9c2f..06801bd02 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -752,7 +752,7 @@ def dao_suspend_service(service_id): .options(joinedload(Service.api_keys)) .filter(Service.id == service_id) ) - service = db.session.execute(stmt).one() + service = db.session.execute(stmt).unique().one() for api_key in service.api_keys: if not api_key.expiry_date: From 5a75a68bb0b3015a58fcb4bc85abf3553f455f30 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 10:37:14 -0700 Subject: [PATCH 067/128] use unique() --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 06801bd02..82d806306 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -752,7 +752,7 @@ def dao_suspend_service(service_id): .options(joinedload(Service.api_keys)) .filter(Service.id == service_id) ) - service = db.session.execute(stmt).unique().one() + service = db.session.execute(stmt).scalars().unique().one() for api_key in service.api_keys: if not api_key.expiry_date: From b77173983a2340256e645855ebd6342a8cb236f7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 10:56:39 -0700 Subject: [PATCH 068/128] fix delete --- app/dao/services_dao.py | 47 ++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 82d806306..e9c40dce9 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -507,39 +507,42 @@ def dao_remove_user_from_service(service, user): def delete_service_and_all_associated_db_objects(service): - def _delete_commit(query): - query.delete(synchronize_session=False) + def _delete_commit(stmt): + # query.delete(synchronize_session=False) + db.session.execute(stmt) db.session.commit() - subq = db.session.query(Template.id).filter_by(service=service).subquery() - _delete_commit( - TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq)) - ) + # subq = db.session.query(Template.id).filter_by(service=service).subquery() + subq = select(Template.id).filter_by(service=service).subquery() - _delete_commit(ServiceSmsSender.query.filter_by(service=service)) - _delete_commit(ServiceEmailReplyTo.query.filter_by(service=service)) - _delete_commit(InvitedUser.query.filter_by(service=service)) - _delete_commit(Permission.query.filter_by(service=service)) - _delete_commit(NotificationHistory.query.filter_by(service=service)) - _delete_commit(Notification.query.filter_by(service=service)) - _delete_commit(Job.query.filter_by(service=service)) - _delete_commit(Template.query.filter_by(service=service)) - _delete_commit(TemplateHistory.query.filter_by(service_id=service.id)) - _delete_commit(ServicePermission.query.filter_by(service_id=service.id)) - _delete_commit(ApiKey.query.filter_by(service=service)) - _delete_commit(ApiKey.get_history_model().query.filter_by(service_id=service.id)) - _delete_commit(AnnualBilling.query.filter_by(service_id=service.id)) + stmt = select(TemplateRedacted).filter(TemplateRedacted.template_id.in_(subq)) + _delete_commit(stmt) - verify_codes = VerifyCode.query.join(User).filter( - User.id.in_([x.id for x in service.users]) + _delete_commit(select(ServiceSmsSender).filter_by(service=service)) + _delete_commit(select(ServiceEmailReplyTo).filter_by(service=service)) + _delete_commit(select(InvitedUser).filter_by(service=service)) + _delete_commit(select(Permission).filter_by(service=service)) + _delete_commit(select(NotificationHistory).filter_by(service=service)) + _delete_commit(select(Notification).filter_by(service=service)) + _delete_commit(select(Job).filter_by(service=service)) + _delete_commit(select(Template).filter_by(service=service)) + _delete_commit(select(TemplateHistory).filter_by(service_id=service.id)) + _delete_commit(select(ServicePermission).filter_by(service_id=service.id)) + _delete_commit(select(ApiKey).filter_by(service=service)) + _delete_commit(select(ApiKey.get_history_model()).filter_by(service_id=service.id)) + _delete_commit(select(AnnualBilling).filter_by(service_id=service.id)) + + stmt = ( + select(VerifyCode).join(User).filter(User.id.in_([x.id for x in service.users])) ) + verify_codes = db.session.execute(stmt).scalar().all() list(map(db.session.delete, verify_codes)) db.session.commit() users = [x for x in service.users] for user in users: user.organizations = [] service.users.remove(user) - _delete_commit(Service.get_history_model().query.filter_by(id=service.id)) + _delete_commit(select(Service.get_history_model()).filter_by(id=service.id)) db.session.delete(service) db.session.commit() for user in users: From 4de9ca5c07b8247190b3bfdf40a4459b504955b0 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 11:08:23 -0700 Subject: [PATCH 069/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e9c40dce9..8bac743dc 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -535,7 +535,7 @@ def delete_service_and_all_associated_db_objects(service): stmt = ( select(VerifyCode).join(User).filter(User.id.in_([x.id for x in service.users])) ) - verify_codes = db.session.execute(stmt).scalar().all() + verify_codes = db.session.execute(stmt).scalars().all() list(map(db.session.delete, verify_codes)) db.session.commit() users = [x for x in service.users] From 464dff64c73830f4b776ba4d9146d753ba284dae Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 11:25:45 -0700 Subject: [PATCH 070/128] fix delete --- app/dao/services_dao.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 8bac743dc..c7f2266c9 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -2,7 +2,7 @@ import uuid from datetime import timedelta from flask import current_app -from sqlalchemy import Float, cast, select +from sqlalchemy import Float, cast, delete, select from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import and_, asc, case, func @@ -515,22 +515,22 @@ def delete_service_and_all_associated_db_objects(service): # subq = db.session.query(Template.id).filter_by(service=service).subquery() subq = select(Template.id).filter_by(service=service).subquery() - stmt = select(TemplateRedacted).filter(TemplateRedacted.template_id.in_(subq)) + stmt = delete(TemplateRedacted).filter(TemplateRedacted.template_id.in_(subq)) _delete_commit(stmt) - _delete_commit(select(ServiceSmsSender).filter_by(service=service)) - _delete_commit(select(ServiceEmailReplyTo).filter_by(service=service)) - _delete_commit(select(InvitedUser).filter_by(service=service)) - _delete_commit(select(Permission).filter_by(service=service)) - _delete_commit(select(NotificationHistory).filter_by(service=service)) - _delete_commit(select(Notification).filter_by(service=service)) - _delete_commit(select(Job).filter_by(service=service)) - _delete_commit(select(Template).filter_by(service=service)) - _delete_commit(select(TemplateHistory).filter_by(service_id=service.id)) - _delete_commit(select(ServicePermission).filter_by(service_id=service.id)) - _delete_commit(select(ApiKey).filter_by(service=service)) - _delete_commit(select(ApiKey.get_history_model()).filter_by(service_id=service.id)) - _delete_commit(select(AnnualBilling).filter_by(service_id=service.id)) + _delete_commit(delete(ServiceSmsSender).filter_by(service=service)) + _delete_commit(delete(ServiceEmailReplyTo).filter_by(service=service)) + _delete_commit(delete(InvitedUser).filter_by(service=service)) + _delete_commit(delete(Permission).filter_by(service=service)) + _delete_commit(delete(NotificationHistory).filter_by(service=service)) + _delete_commit(delete(Notification).filter_by(service=service)) + _delete_commit(delete(Job).filter_by(service=service)) + _delete_commit(delete(Template).filter_by(service=service)) + _delete_commit(delete(TemplateHistory).filter_by(service_id=service.id)) + _delete_commit(delete(ServicePermission).filter_by(service_id=service.id)) + _delete_commit(delete(ApiKey).filter_by(service=service)) + _delete_commit(delete(ApiKey.get_history_model()).filter_by(service_id=service.id)) + _delete_commit(delete(AnnualBilling).filter_by(service_id=service.id)) stmt = ( select(VerifyCode).join(User).filter(User.id.in_([x.id for x in service.users])) @@ -542,7 +542,7 @@ def delete_service_and_all_associated_db_objects(service): for user in users: user.organizations = [] service.users.remove(user) - _delete_commit(select(Service.get_history_model()).filter_by(id=service.id)) + _delete_commit(delete(Service.get_history_model()).filter_by(id=service.id)) db.session.delete(service) db.session.commit() for user in users: From 3f2354909547cd5340ac02f6ef231fce10207916 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 11:45:26 -0700 Subject: [PATCH 071/128] fix delete --- app/dao/services_dao.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index c7f2266c9..f1e934bcf 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -695,7 +695,7 @@ def dao_fetch_todays_stats_for_all_services( end_date = get_midnight_in_utc(today + timedelta(days=1)) subquery = ( - db.session.query( + select( Notification.notification_type, Notification.status, Notification.service_id, @@ -714,8 +714,8 @@ def dao_fetch_todays_stats_for_all_services( subquery = subquery.subquery() - query = ( - db.session.query( + stmt = ( + select( Service.id.label("service_id"), Service.name, Service.restricted, @@ -730,9 +730,9 @@ def dao_fetch_todays_stats_for_all_services( ) if only_active: - query = query.filter(Service.active) + stmt = stmt.filter(Service.active) - return query.all() + return db.session.execute(stmt).scalars().all() @autocommit @@ -830,7 +830,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500) def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000): subquery = ( - db.session.query( + select( func.count(Notification.id).label("total_count"), Notification.service_id.label("service_id"), ) @@ -851,8 +851,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 subquery = subquery.subquery() - query = ( - db.session.query( + stmt = ( + select( Notification.service_id.label("service_id"), func.count(Notification.id).label("permanent_failure_count"), subquery.c.total_count.label("total_count"), @@ -880,7 +880,7 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 ) ) - return query.all() + return db.session.execute(stmt).scalars().all() def get_live_services_with_organization(): From 337b7becc979af81c8cb7871a55dff2f9979e0c8 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 11:57:42 -0700 Subject: [PATCH 072/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index f1e934bcf..e39d8ee46 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -633,7 +633,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): func.date_trunc("day", NotificationAllTimeView.created_at), ) ) - return db.session.execute(stmt).all() + return db.session.execute(stmt).scalars().all() def dao_fetch_stats_for_service_from_days_for_user( From 190c3cdbb5597f1a15217d08f4f0ce61a0ad4132 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 12:10:45 -0700 Subject: [PATCH 073/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e39d8ee46..cabd991d3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -586,7 +586,7 @@ def dao_fetch_todays_stats_for_service(service_id): Notification.status, ) ) - return db.session.execute(stmt).all() + return db.session.execute(stmt).scalars().all() def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): From a4f0ff744014fb1fc7d872ef16f8cf9fbd2e6763 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 12:38:23 -0700 Subject: [PATCH 074/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index cabd991d3..d08a69885 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -732,7 +732,7 @@ def dao_fetch_todays_stats_for_all_services( if only_active: stmt = stmt.filter(Service.active) - return db.session.execute(stmt).scalars().all() + return db.session.execute(stmt).all() @autocommit From c97ea3be1c671c2ee271074c120ad01a19b1141c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 12:55:08 -0700 Subject: [PATCH 075/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index d08a69885..9f1a52182 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -586,7 +586,7 @@ def dao_fetch_todays_stats_for_service(service_id): Notification.status, ) ) - return db.session.execute(stmt).scalars().all() + return db.session.execute(stmt).all() def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): From 7eb58e9e4c0a5b0c3131e5b327c29b7b7184d026 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 13:04:28 -0700 Subject: [PATCH 076/128] fix delete --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 9f1a52182..585fe83b1 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -880,7 +880,7 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 ) ) - return db.session.execute(stmt).scalars().all() + return db.session.execute(stmt).all() def get_live_services_with_organization(): From 6114fe0705b6254779c07eb331ca2f7204620ff9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 13:35:34 -0700 Subject: [PATCH 077/128] try to fix test code --- .ds.baseline | 4 +- tests/app/dao/test_services_dao.py | 100 +++++++++++++++++------------ 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 1c279e018..4ac79c91f 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 265, + "line_number": 282, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-09-27T16:42:53Z" + "generated_at": "2024-10-10T20:35:29Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index e590eb5b4..83a6408f6 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -6,6 +6,7 @@ from unittest.mock import Mock import pytest import sqlalchemy from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound @@ -89,9 +90,25 @@ from tests.app.db import ( ) +def _get_service_query_count(): + stmt = select(func.count(Service.id)) + return db.session.execute(stmt).scalar() + + +def _get_first_service(): + stmt = select(Service).limit(1) + service = db.session.execute(stmt).scalars().first() + return service + + +def _get_service_by_id(service_id): + stmt = select(Service).where(id == service_id) + return db.session.execute(stmt).one() + + def test_create_service(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -101,8 +118,8 @@ def test_create_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 - service_db = Service.query.one() + assert _get_service_query_count() == 1 + service_db = _get_first_service() assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -120,7 +137,7 @@ def test_create_service_with_organization(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 service = Service( name="service_name", email_from="email_from", @@ -130,8 +147,8 @@ def test_create_service_with_organization(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 - service_db = Service.query.one() + assert _get_service_query_count()() == 1 + service_db = _get_first_service() organization = Organization.query.get(organization.id) assert service_db.name == "service_name" assert service_db.id == service.id @@ -151,7 +168,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 service = Service( name="service_name", email_from="email_from", @@ -161,8 +178,8 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 - service_db = Service.query.one() + assert _get_service_query_count() == 1 + service_db = _get_first_service() organization = Organization.query.get(organization.id) assert service_db.name == "service_name" assert service_db.id == service.id @@ -183,7 +200,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): def test_cannot_create_two_services_with_same_name(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 service1 = Service( name="service_name", email_from="email_from1", @@ -209,7 +226,7 @@ def test_cannot_create_two_services_with_same_name(notify_db_session): def test_cannot_create_two_services_with_same_email_from(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 service1 = Service( name="service_name1", email_from="email_from", @@ -235,7 +252,7 @@ def test_cannot_create_two_services_with_same_email_from(notify_db_session): def test_cannot_create_service_with_no_user(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 service = Service( name="service_name", email_from="email_from", @@ -258,7 +275,7 @@ def test_should_add_user_to_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert user in Service.query.first().users + assert user in _get_first_service().users new_user = User( name="Test User", email_address="new_user@digital.fake.gov", @@ -267,7 +284,7 @@ def test_should_add_user_to_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in Service.query.first().users + assert new_user in _get_first_service().users def test_dao_add_user_to_service_sets_folder_permissions(sample_user, sample_service): @@ -347,9 +364,9 @@ def test_should_remove_user_from_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in Service.query.first().users + assert new_user in _get_first_service().users dao_remove_user_from_service(service, new_user) - assert new_user not in Service.query.first().users + assert new_user not in _get_first_service().users def test_should_remove_user_from_service_exception(notify_db_session): @@ -668,7 +685,7 @@ def test_removing_all_permission_returns_service_with_no_permissions(notify_db_s def test_create_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -678,10 +695,10 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 + assert _get_service_query_count()() == 1 assert Service.get_history_model().query.count() == 1 - service_from_db = Service.query.first() + service_from_db = _get_first_service() service_history = Service.get_history_model().query.first() assert service_from_db.id == service_history.id @@ -694,7 +711,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses def test_update_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -705,17 +722,17 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses ) dao_create_service(service, user) - assert Service.query.count() == 1 - assert Service.query.first().version == 1 + assert _get_service_query_count() == 1 + assert _get_first_service().version == 1 assert Service.get_history_model().query.count() == 1 service.name = "updated_service_name" dao_update_service(service) - assert Service.query.count() == 1 + assert _get_service_query_count()() == 1 assert Service.get_history_model().query.count() == 2 - service_from_db = Service.query.first() + service_from_db = _get_first_service() assert service_from_db.version == 2 @@ -736,7 +753,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( notify_db_session, ): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -755,17 +772,17 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ], ) - assert Service.query.count() == 1 + assert _get_service_query_count()() == 1 service.permissions.append( ServicePermission(service_id=service.id, permission=ServicePermissionType.EMAIL) ) dao_update_service(service) - assert Service.query.count() == 1 + assert _get_service_query_count()() == 1 assert Service.get_history_model().query.count() == 2 - service_from_db = Service.query.first() + service_from_db = _get_first_service() assert service_from_db.version == 2 @@ -784,10 +801,10 @@ def test_update_service_permission_creates_a_history_record_with_current_data( service.permissions.remove(permission) dao_update_service(service) - assert Service.query.count() == 1 + assert _get_service_query_count()() == 1 assert Service.get_history_model().query.count() == 3 - service_from_db = Service.query.first() + service_from_db = _get_first_service() assert service_from_db.version == 3 _assert_service_permissions( service.permissions, @@ -797,20 +814,19 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ), ) - history = ( - Service.get_history_model() - .query.filter_by(name="service_name") + stmt = ( + select(Service.get_history_model()) + .filter_by(name="service_name") .order_by("version") - .all() ) - + history = db.session.execute(stmt).all() assert len(history) == 3 assert history[2].version == 3 def test_create_service_and_history_is_transactional(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name=None, @@ -828,7 +844,7 @@ def test_create_service_and_history_is_transactional(notify_db_session): in str(seeei) ) - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 assert Service.get_history_model().query.count() == 0 @@ -865,7 +881,7 @@ def test_delete_service_and_associated_objects(notify_db_session): assert Permission.query.count() == 0 assert User.query.count() == 0 assert InvitedUser.query.count() == 0 - assert Service.query.count() == 0 + assert _get_service_query_count()() == 0 assert Service.get_history_model().query.count() == 0 assert ServicePermission.query.count() == 0 # the organization hasn't been deleted @@ -1316,7 +1332,7 @@ def test_dao_fetch_todays_stats_for_all_services_can_exclude_from_test_key( def test_dao_suspend_service_with_no_api_keys(notify_db_session): service = create_service() dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = _get_service_by_id(service.id) assert not service.active assert service.name == service.name assert service.api_keys == [] @@ -1329,7 +1345,7 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys( service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = _get_service_by_id(service.id) assert not service.active assert service.name == service.name @@ -1344,11 +1360,11 @@ def test_dao_resume_service_marks_service_as_active_and_api_keys_are_still_revok service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = _get_service_by_id(service.id) assert not service.active dao_resume_service(service.id) - assert Service.query.get(service.id).active + assert _get_service_by_id(service.id).active api_key = ApiKey.query.get(api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) From f3a1139a85786385821864680473539abb61e1f7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 13:41:33 -0700 Subject: [PATCH 078/128] try to fix test code --- tests/app/dao/test_services_dao.py | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 83a6408f6..810fdc91f 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -137,7 +137,7 @@ def test_create_service_with_organization(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -147,7 +147,7 @@ def test_create_service_with_organization(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 service_db = _get_first_service() organization = Organization.query.get(organization.id) assert service_db.name == "service_name" @@ -168,7 +168,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -200,7 +200,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): def test_cannot_create_two_services_with_same_name(notify_db_session): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 service1 = Service( name="service_name", email_from="email_from1", @@ -226,7 +226,7 @@ def test_cannot_create_two_services_with_same_name(notify_db_session): def test_cannot_create_two_services_with_same_email_from(notify_db_session): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 service1 = Service( name="service_name1", email_from="email_from", @@ -252,7 +252,7 @@ def test_cannot_create_two_services_with_same_email_from(notify_db_session): def test_cannot_create_service_with_no_user(notify_db_session): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -685,7 +685,7 @@ def test_removing_all_permission_returns_service_with_no_permissions(notify_db_s def test_create_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -695,7 +695,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses created_by=user, ) dao_create_service(service, user) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 assert Service.get_history_model().query.count() == 1 service_from_db = _get_first_service() @@ -711,7 +711,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses def test_update_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -729,7 +729,7 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses service.name = "updated_service_name" dao_update_service(service) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 assert Service.get_history_model().query.count() == 2 service_from_db = _get_first_service() @@ -753,7 +753,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( notify_db_session, ): user = create_user() - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", @@ -772,14 +772,14 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ], ) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 service.permissions.append( ServicePermission(service_id=service.id, permission=ServicePermissionType.EMAIL) ) dao_update_service(service) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 assert Service.get_history_model().query.count() == 2 service_from_db = _get_first_service() @@ -801,7 +801,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( service.permissions.remove(permission) dao_update_service(service) - assert _get_service_query_count()() == 1 + assert _get_service_query_count() == 1 assert Service.get_history_model().query.count() == 3 service_from_db = _get_first_service() @@ -844,7 +844,7 @@ def test_create_service_and_history_is_transactional(notify_db_session): in str(seeei) ) - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 @@ -881,7 +881,7 @@ def test_delete_service_and_associated_objects(notify_db_session): assert Permission.query.count() == 0 assert User.query.count() == 0 assert InvitedUser.query.count() == 0 - assert _get_service_query_count()() == 0 + assert _get_service_query_count() == 0 assert Service.get_history_model().query.count() == 0 assert ServicePermission.query.count() == 0 # the organization hasn't been deleted From 5c818b520ce03d6279a8aa1aaf1e756911212c02 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 13:52:29 -0700 Subject: [PATCH 079/128] try to fix test code --- tests/app/dao/test_services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 810fdc91f..25c6bd992 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -102,8 +102,8 @@ def _get_first_service(): def _get_service_by_id(service_id): - stmt = select(Service).where(id == service_id) - return db.session.execute(stmt).one() + stmt = select(Service).filter(id == service_id) + return db.session.execute(stmt).scalars().one() def test_create_service(notify_db_session): From 58d9b2d63ea0309f7ea8392ccad1e52baa50b652 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 14:03:11 -0700 Subject: [PATCH 080/128] try to fix test code --- .ds.baseline | 4 ++-- tests/app/dao/test_services_dao.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 4ac79c91f..01fb2bf3d 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 282, + "line_number": 285, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-10T20:35:29Z" + "generated_at": "2024-10-10T21:03:06Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 25c6bd992..a7c2f7e77 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -102,8 +102,11 @@ def _get_first_service(): def _get_service_by_id(service_id): - stmt = select(Service).filter(id == service_id) - return db.session.execute(stmt).scalars().one() + stmt = select(Service).filter(Service.id == service_id) + + service = db.session.execute(stmt).scalars().one() + print(f"SERVICE Is {service}") + return service def test_create_service(notify_db_session): From b385e0863f8c5d792fc46b8df504ffbac073546a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 14:14:35 -0700 Subject: [PATCH 081/128] try to fix test code --- tests/app/dao/test_services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index a7c2f7e77..7e01e8e1e 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -822,7 +822,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( .filter_by(name="service_name") .order_by("version") ) - history = db.session.execute(stmt).all() + history = db.session.execute(stmt).scalars().all() assert len(history) == 3 assert history[2].version == 3 From 97237f8e5da8af59308a90dc9526c78cc788c43a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 14:35:43 -0700 Subject: [PATCH 082/128] try to fix test code --- .ds.baseline | 4 +- tests/app/dao/test_services_dao.py | 69 +++++++++++++++++++----------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 01fb2bf3d..ee20a155d 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 285, + "line_number": 290, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-10T21:03:06Z" + "generated_at": "2024-10-10T21:35:38Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 7e01e8e1e..df08ce06e 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -95,6 +95,11 @@ def _get_service_query_count(): return db.session.execute(stmt).scalar() +def _get_service_history_query_count(): + stmt = select(Service.get_history_model()) + return db.session.execute(stmt).scalar() + + def _get_first_service(): stmt = select(Service).limit(1) service = db.session.execute(stmt).scalars().first() @@ -152,7 +157,7 @@ def test_create_service_with_organization(notify_db_session): dao_create_service(service, user) assert _get_service_query_count() == 1 service_db = _get_first_service() - organization = Organization.query.get(organization.id) + organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -183,7 +188,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): dao_create_service(service, user) assert _get_service_query_count() == 1 service_db = _get_first_service() - organization = Organization.query.get(organization.id) + organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -689,7 +694,7 @@ def test_removing_all_permission_returns_service_with_no_permissions(notify_db_s def test_create_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 + assert _get_service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -699,7 +704,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses ) dao_create_service(service, user) assert _get_service_query_count() == 1 - assert Service.get_history_model().query.count() == 1 + assert _get_service_history_query_count() == 1 service_from_db = _get_first_service() service_history = Service.get_history_model().query.first() @@ -715,7 +720,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses def test_update_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 + assert _get_service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -727,13 +732,13 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses assert _get_service_query_count() == 1 assert _get_first_service().version == 1 - assert Service.get_history_model().query.count() == 1 + assert _get_service_history_query_count() == 1 service.name = "updated_service_name" dao_update_service(service) assert _get_service_query_count() == 1 - assert Service.get_history_model().query.count() == 2 + assert _get_service_history_query_count() == 2 service_from_db = _get_first_service() @@ -757,7 +762,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ): user = create_user() assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 + assert _get_service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -783,7 +788,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( dao_update_service(service) assert _get_service_query_count() == 1 - assert Service.get_history_model().query.count() == 2 + assert _get_service_history_query_count() == 2 service_from_db = _get_first_service() @@ -805,7 +810,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( dao_update_service(service) assert _get_service_query_count() == 1 - assert Service.get_history_model().query.count() == 3 + assert _get_service_history_query_count() == 3 service_from_db = _get_first_service() assert service_from_db.version == 3 @@ -830,7 +835,7 @@ def test_update_service_permission_creates_a_history_record_with_current_data( def test_create_service_and_history_is_transactional(notify_db_session): user = create_user() assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 + assert _get_service_history_query_count() == 0 service = Service( name=None, email_from="email_from", @@ -848,7 +853,7 @@ def test_create_service_and_history_is_transactional(notify_db_session): ) assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 + assert _get_service_history_query_count() == 0 def test_delete_service_and_associated_objects(notify_db_session): @@ -874,21 +879,35 @@ def test_delete_service_and_associated_objects(notify_db_session): ) delete_service_and_all_associated_db_objects(service) - assert VerifyCode.query.count() == 0 - assert ApiKey.query.count() == 0 - assert ApiKey.get_history_model().query.count() == 0 - assert Template.query.count() == 0 - assert TemplateHistory.query.count() == 0 - assert Job.query.count() == 0 - assert Notification.query.count() == 0 - assert Permission.query.count() == 0 - assert User.query.count() == 0 - assert InvitedUser.query.count() == 0 + stmt = select(VerifyCode) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(ApiKey) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(ApiKey.get_history_model()) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(Template) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(TemplateHistory) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(Job) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(Notification) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(Permission) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(User) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(InvitedUser) + assert db.session.execute(stmt).scalar() == 0 + assert _get_service_query_count() == 0 - assert Service.get_history_model().query.count() == 0 - assert ServicePermission.query.count() == 0 + assert _get_service_history_query_count() == 0 + stmt = select(ServicePermission) + assert db.session.execute(stmt).scalar() == 0 + # the organization hasn't been deleted - assert Organization.query.count() == 1 + stmt = select(Organization) + assert db.session.execute(stmt).scalar() == 1 def test_add_existing_user_to_another_service_doesnot_change_old_permissions( From 79d666f57037854d847868ff56f8bbc1d342fd3d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 14:55:39 -0700 Subject: [PATCH 083/128] try to fix test code --- .ds.baseline | 4 ++-- tests/app/dao/test_services_dao.py | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index ee20a155d..ae7f12d3e 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 290, + "line_number": 289, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-10T21:35:38Z" + "generated_at": "2024-10-10T21:55:35Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index df08ce06e..ca7fd67ba 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -92,12 +92,12 @@ from tests.app.db import ( def _get_service_query_count(): stmt = select(func.count(Service.id)) - return db.session.execute(stmt).scalar() + return db.session.execute(stmt).scalar() or 0 def _get_service_history_query_count(): stmt = select(Service.get_history_model()) - return db.session.execute(stmt).scalar() + return db.session.execute(stmt).scalar() or 0 def _get_first_service(): @@ -110,7 +110,6 @@ def _get_service_by_id(service_id): stmt = select(Service).filter(Service.id == service_id) service = db.session.execute(stmt).scalars().one() - print(f"SERVICE Is {service}") return service @@ -880,30 +879,30 @@ def test_delete_service_and_associated_objects(notify_db_session): delete_service_and_all_associated_db_objects(service) stmt = select(VerifyCode) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(ApiKey) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(ApiKey.get_history_model()) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(Template) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(TemplateHistory) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(Job) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(Notification) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(Permission) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(User) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None stmt = select(InvitedUser) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None assert _get_service_query_count() == 0 assert _get_service_history_query_count() == 0 stmt = select(ServicePermission) - assert db.session.execute(stmt).scalar() == 0 + assert db.session.execute(stmt).scalar() is None # the organization hasn't been deleted stmt = select(Organization) From 4302610d259a87d46598215b6d0187060e57ba14 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 10 Oct 2024 15:08:00 -0700 Subject: [PATCH 084/128] try to fix test code --- tests/app/dao/test_services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index ca7fd67ba..df642e173 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -96,7 +96,7 @@ def _get_service_query_count(): def _get_service_history_query_count(): - stmt = select(Service.get_history_model()) + stmt = select(func.count(Service.get_history_model().id)) return db.session.execute(stmt).scalar() or 0 @@ -905,7 +905,7 @@ def test_delete_service_and_associated_objects(notify_db_session): assert db.session.execute(stmt).scalar() is None # the organization hasn't been deleted - stmt = select(Organization) + stmt = select(func.count(Organization.id)) assert db.session.execute(stmt).scalar() == 1 From 6da190c1ed488a621c06bb96fb1d53371c5fd28e Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 07:07:10 -0700 Subject: [PATCH 085/128] try to fix test code --- tests/app/dao/test_services_dao.py | 71 +++++++++++++----------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index df642e173..2efba7472 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -338,7 +338,8 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a other_service_folder = create_template_folder(other_service) folder_permissions = [str(other_service_folder.id)] - assert ServiceUser.query.count() == 2 + stmt = select(func.count(ServiceUser.id)) + assert db.session.execute(stmt).scalar() == 2 with pytest.raises(IntegrityError) as e: dao_add_user_to_service( @@ -350,7 +351,8 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a 'insert or update on table "user_folder_permissions" violates foreign key constraint' in str(e.value) ) - assert ServiceUser.query.count() == 2 + stmt = select(func.count(ServiceUser.id)) + assert db.session.execute(stmt).scalar() == 2 def test_should_remove_user_from_service(notify_db_session): @@ -406,11 +408,12 @@ def test_should_remove_user_from_service_exception(notify_db_session): def test_removing_a_user_from_a_service_deletes_their_permissions( sample_user, sample_service ): - assert len(Permission.query.all()) == 7 + stmt = select(Permission) + assert len(db.session.execute(stmt).all()) == 7 dao_remove_user_from_service(sample_service, sample_user) - assert Permission.query.all() == [] + assert db.session.execute(stmt).all() == [] def test_removing_a_user_from_a_service_deletes_their_folder_permissions_for_that_service( @@ -706,7 +709,8 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses assert _get_service_history_query_count() == 1 service_from_db = _get_first_service() - service_history = Service.get_history_model().query.first() + stmt = select(Service.get_history_model()) + service_history = db.session.execute(stmt).first() assert service_from_db.id == service_history.id assert service_from_db.name == service_history.name @@ -742,18 +746,10 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses service_from_db = _get_first_service() assert service_from_db.version == 2 - - assert ( - Service.get_history_model().query.filter_by(name="service_name").one().version - == 1 - ) - assert ( - Service.get_history_model() - .query.filter_by(name="updated_service_name") - .one() - .version - == 2 - ) + stmt = select(Service.get_history_model()).filter_by(name="service_name") + assert db.session.execute(stmt).one().version == 1 + stmt = select(Service.get_history_model()).filter_by(name="updated_service_name") + assert db.session.execute(stmt).one().version == 2 def test_update_service_permission_creates_a_history_record_with_current_data( @@ -868,8 +864,8 @@ def test_delete_service_and_associated_objects(notify_db_session): create_notification(template=template, api_key=api_key) create_invited_user(service=service) user.organizations = [organization] - - assert ServicePermission.query.count() == len( + stmt = select(func.count(ServicePermission.id)) + assert db.session.execute(stmt).scalar() == len( ( ServicePermissionType.SMS, ServicePermissionType.EMAIL, @@ -924,9 +920,8 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_one, user) assert user.id == service_one.users[0].id - test_user_permissions = Permission.query.filter_by( - service=service_one, user=user - ).all() + stmt = select(Permission).filter_by(service=service_one, user=user) + test_user_permissions = db.session.execute(stmt).all() assert len(test_user_permissions) == 7 other_user = User( @@ -946,14 +941,12 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_two, other_user) assert other_user.id == service_two.users[0].id - other_user_permissions = Permission.query.filter_by( - service=service_two, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_two, user=other_user) + other_user_permissions = db.session.execute(stmt).all() assert len(other_user_permissions) == 7 + stmt = select(Permission).filter_by(service=service_one, user=other_user) + other_user_service_one_permissions = db.session.execute(stmt).all() - other_user_service_one_permissions = Permission.query.filter_by( - service=service_one, user=other_user - ).all() assert len(other_user_service_one_permissions) == 0 # adding the other_user to service_one should leave all other_user permissions on service_two intact @@ -962,15 +955,12 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( permissions.append(Permission(permission=p)) dao_add_user_to_service(service_one, other_user, permissions=permissions) - - other_user_service_one_permissions = Permission.query.filter_by( - service=service_one, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_one, user=other_user) + other_user_service_one_permissions = db.session.execute(stmt).all() assert len(other_user_service_one_permissions) == 2 - other_user_service_two_permissions = Permission.query.filter_by( - service=service_two, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_two, user=other_user) + other_user_service_two_permissions = db.session.execute(stmt).all() assert len(other_user_service_two_permissions) == 7 @@ -993,9 +983,10 @@ def test_fetch_stats_filters_on_service(notify_db_session): def test_fetch_stats_ignores_historical_notification_data(sample_template): create_notification_history(template=sample_template) - - assert Notification.query.count() == 0 - assert NotificationHistory.query.count() == 1 + stmt = select(func.count(Notification.id)) + assert db.session.execute(stmt).scalar() == 0 + stmt = select(func.count(NotificationHistory.id)) + assert db.session.execute(stmt).scalar() == 1 stats = dao_fetch_todays_stats_for_service(sample_template.service_id) assert len(stats) == 0 @@ -1370,7 +1361,7 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys( assert not service.active assert service.name == service.name - api_key = ApiKey.query.get(api_key.id) + api_key = db.session.get(ApiKey, api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) @@ -1387,7 +1378,7 @@ def test_dao_resume_service_marks_service_as_active_and_api_keys_are_still_revok dao_resume_service(service.id) assert _get_service_by_id(service.id).active - api_key = ApiKey.query.get(api_key.id) + api_key = db.session.get(ApiKey, api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) From 3021677d9e8a4982dd68366f04674ddfe85f6634 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 07:21:40 -0700 Subject: [PATCH 086/128] try to fix test code --- tests/app/dao/test_services_dao.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 2efba7472..613fc21df 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -338,7 +338,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a other_service_folder = create_template_folder(other_service) folder_permissions = [str(other_service_folder.id)] - stmt = select(func.count(ServiceUser.id)) + stmt = select(func.count(ServiceUser.service_id)) assert db.session.execute(stmt).scalar() == 2 with pytest.raises(IntegrityError) as e: @@ -351,7 +351,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a 'insert or update on table "user_folder_permissions" violates foreign key constraint' in str(e.value) ) - stmt = select(func.count(ServiceUser.id)) + stmt = select(func.count(ServiceUser.service_id)) assert db.session.execute(stmt).scalar() == 2 @@ -864,7 +864,7 @@ def test_delete_service_and_associated_objects(notify_db_session): create_notification(template=template, api_key=api_key) create_invited_user(service=service) user.organizations = [organization] - stmt = select(func.count(ServicePermission.id)) + stmt = select(func.count(ServicePermission.service_id)) assert db.session.execute(stmt).scalar() == len( ( ServicePermissionType.SMS, From b52070ff3302d55e2fe89122337ca9c328c04145 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 07:30:34 -0700 Subject: [PATCH 087/128] try to fix test code --- tests/app/dao/test_services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 613fc21df..8c0b5e452 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -747,7 +747,7 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses assert service_from_db.version == 2 stmt = select(Service.get_history_model()).filter_by(name="service_name") - assert db.session.execute(stmt).one().version == 1 + assert db.session.execute(stmt).scalars().one().version == 1 stmt = select(Service.get_history_model()).filter_by(name="updated_service_name") assert db.session.execute(stmt).one().version == 2 From fa5af8e02cea56cb7c970ad2e7aa33a6bfe7a40b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 07:40:29 -0700 Subject: [PATCH 088/128] try to fix test code --- tests/app/dao/test_services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 8c0b5e452..61fe99419 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -710,7 +710,7 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses service_from_db = _get_first_service() stmt = select(Service.get_history_model()) - service_history = db.session.execute(stmt).first() + service_history = db.session.execute(stmt).scalars().first() assert service_from_db.id == service_history.id assert service_from_db.name == service_history.name @@ -749,7 +749,7 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses stmt = select(Service.get_history_model()).filter_by(name="service_name") assert db.session.execute(stmt).scalars().one().version == 1 stmt = select(Service.get_history_model()).filter_by(name="updated_service_name") - assert db.session.execute(stmt).one().version == 2 + assert db.session.execute(stmt).scalars().one().version == 2 def test_update_service_permission_creates_a_history_record_with_current_data( From 3adedc535172252d6f5978e9cef8c1c92e49af3c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 07:53:47 -0700 Subject: [PATCH 089/128] try to fix test code --- .ds.baseline | 6 +++--- tests/app/dao/test_users_dao.py | 27 +++++++++++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index ae7f12d3e..0d0277bfc 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -249,7 +249,7 @@ "filename": "tests/app/dao/test_users_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 52, + "line_number": 54, "is_secret": false }, { @@ -257,7 +257,7 @@ "filename": "tests/app/dao/test_users_dao.py", "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba", "is_verified": false, - "line_number": 176, + "line_number": 184, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-10T21:55:35Z" + "generated_at": "2024-10-11T14:53:42Z" } diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index 9c8770913..e5fd8a8c1 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -3,6 +3,7 @@ from datetime import timedelta import pytest from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.exc import DataError from sqlalchemy.orm.exc import NoResultFound @@ -55,8 +56,10 @@ def test_create_user(notify_db_session, phone_number, expected_phone_number): } user = User(**data) save_model_user(user, password="password", validated_email_access=True) - assert User.query.count() == 1 - user_query = User.query.first() + stmt = select(func.count(User.id)) + assert db.session.execute(stmt).scalar() == 1 + stmt = select(User) + user_query = db.session.execute(stmt).scalars().first() assert user_query.email_address == email assert user_query.id == user.id assert user_query.mobile_number == expected_phone_number @@ -68,7 +71,8 @@ def test_get_all_users(notify_db_session): create_user(email="1@test.com") create_user(email="2@test.com") - assert User.query.count() == 2 + stmt = select(func.count(User.id)) + assert db.session.execute(stmt).scalar() == 2 assert len(get_user_by_id()) == 2 @@ -89,9 +93,10 @@ def test_get_user_invalid_id(notify_db_session): def test_delete_users(sample_user): - assert User.query.count() == 1 + stmt = select(func.count(User.id)) + assert db.session.execute(stmt).scalar() == 1 delete_model_user(sample_user) - assert User.query.count() == 0 + assert db.session.execute(stmt).scalar() == 0 def test_increment_failed_login_should_increment_failed_logins(sample_user): @@ -127,9 +132,10 @@ def test_get_user_by_email_is_case_insensitive(sample_user): def test_should_delete_all_verification_codes_more_than_one_day_old(sample_user): make_verify_code(sample_user, age=timedelta(hours=24), code="54321") make_verify_code(sample_user, age=timedelta(hours=24), code="54321") - assert VerifyCode.query.count() == 2 + stmt = select(func.count(VerifyCode.id)) + assert db.session.execute(stmt).scalar() == 2 delete_codes_older_created_more_than_a_day_ago() - assert VerifyCode.query.count() == 0 + assert db.session.execute(stmt).scalar() == 0 def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user): @@ -137,10 +143,11 @@ def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user) sample_user, age=timedelta(hours=23, minutes=59, seconds=59), code="12345" ) make_verify_code(sample_user, age=timedelta(hours=24), code="54321") - - assert VerifyCode.query.count() == 2 + stmt = select(func.count(VerifyCode)) + assert db.session.execute(stmt).scalar() == 2 delete_codes_older_created_more_than_a_day_ago() - assert VerifyCode.query.one()._code == "12345" + stmt = select(VerifyCode) + assert db.session.execute(stmt).scalars().one()._code == "12345" def make_verify_code(user, age=None, expiry_age=None, code="12335", code_used=False): From ad1ef5bfd2139e19e3850f3e004a3cb8c9ebf4ed Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 08:07:22 -0700 Subject: [PATCH 090/128] try to fix test code --- tests/app/dao/test_service_sms_sender_dao.py | 29 +++++++++++--------- tests/app/dao/test_users_dao.py | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/app/dao/test_service_sms_sender_dao.py b/tests/app/dao/test_service_sms_sender_dao.py index 9ca05e711..bd0bb8801 100644 --- a/tests/app/dao/test_service_sms_sender_dao.py +++ b/tests/app/dao/test_service_sms_sender_dao.py @@ -1,6 +1,7 @@ import uuid import pytest +from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError from app.dao.service_sms_sender_dao import ( @@ -13,6 +14,7 @@ from app.dao.service_sms_sender_dao import ( ) from app.exceptions import ArchiveValidationError from app.models import ServiceSmsSender +from tests.app import db from tests.app.db import ( create_inbound_number, create_service, @@ -97,10 +99,8 @@ def test_dao_add_sms_sender_for_service(notify_db_session): is_default=False, inbound_number_id=None, ) - - service_sms_senders = ServiceSmsSender.query.order_by( - ServiceSmsSender.created_at - ).all() + stmt = select(ServiceSmsSender).order_by(ServiceSmsSender.created_at) + service_sms_senders = db.session.execute(stmt).scalars().all() assert len(service_sms_senders) == 2 assert service_sms_senders[0].sms_sender == "testing" assert service_sms_senders[0].is_default @@ -116,10 +116,8 @@ def test_dao_add_sms_sender_for_service_switches_default(notify_db_session): is_default=True, inbound_number_id=None, ) - - service_sms_senders = ServiceSmsSender.query.order_by( - ServiceSmsSender.created_at - ).all() + stmt = select(ServiceSmsSender).order_by(ServiceSmsSender.created_at) + service_sms_senders = db.session.execute(stmt).scalars().all() assert len(service_sms_senders) == 2 assert service_sms_senders[0].sms_sender == "testing" assert not service_sms_senders[0].is_default @@ -128,7 +126,8 @@ def test_dao_add_sms_sender_for_service_switches_default(notify_db_session): def test_dao_update_service_sms_sender(notify_db_session): service = create_service() - service_sms_senders = ServiceSmsSender.query.filter_by(service_id=service.id).all() + stmt = select(ServiceSmsSender).filter_by(service_id=service.id) + service_sms_senders = db.session.execute(stmt).scalars().all() assert len(service_sms_senders) == 1 sms_sender_to_update = service_sms_senders[0] @@ -138,7 +137,8 @@ def test_dao_update_service_sms_sender(notify_db_session): is_default=True, sms_sender="updated", ) - sms_senders = ServiceSmsSender.query.filter_by(service_id=service.id).all() + stmt = select(ServiceSmsSender).filter_by(service_id=service.id) + sms_senders = db.sessions.execute(stmt).scalars().all() assert len(sms_senders) == 1 assert sms_senders[0].is_default assert sms_senders[0].sms_sender == "updated" @@ -159,7 +159,8 @@ def test_dao_update_service_sms_sender_switches_default(notify_db_session): is_default=True, sms_sender="updated", ) - sms_senders = ServiceSmsSender.query.filter_by(service_id=service.id).all() + stmt = select(ServiceSmsSender).filter_by(service_id=service.id) + sms_senders = db.session.execute(stmt).scalars().all() expected = {("testing", False), ("updated", True)} results = {(sender.sms_sender, sender.is_default) for sender in sms_senders} @@ -190,7 +191,8 @@ def test_update_existing_sms_sender_with_inbound_number(notify_db_session): service = create_service() inbound_number = create_inbound_number(number="12345", service_id=service.id) - existing_sms_sender = ServiceSmsSender.query.filter_by(service_id=service.id).one() + stmt = select(ServiceSmsSender).filter_by(service_id=service.id) + existing_sms_sender = db.session.execute(stmt).scalars().one() sms_sender = update_existing_sms_sender_with_inbound_number( service_sms_sender=existing_sms_sender, sms_sender=inbound_number.number, @@ -206,7 +208,8 @@ def test_update_existing_sms_sender_with_inbound_number_raises_exception_if_inbo notify_db_session, ): service = create_service() - existing_sms_sender = ServiceSmsSender.query.filter_by(service_id=service.id).one() + stmt = select(ServiceSmsSender).filter_by(service_id=service.id) + existing_sms_sender = db.session.execute(stmt).scalars().one() with pytest.raises(expected_exception=SQLAlchemyError): update_existing_sms_sender_with_inbound_number( service_sms_sender=existing_sms_sender, diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index e5fd8a8c1..3636593f9 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -143,7 +143,7 @@ def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user) sample_user, age=timedelta(hours=23, minutes=59, seconds=59), code="12345" ) make_verify_code(sample_user, age=timedelta(hours=24), code="54321") - stmt = select(func.count(VerifyCode)) + stmt = select(func.count(VerifyCode.id)) assert db.session.execute(stmt).scalar() == 2 delete_codes_older_created_more_than_a_day_ago() stmt = select(VerifyCode) From aac64b2888443a2dfb0caed3b00b1d6f87604b8b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 08:39:55 -0700 Subject: [PATCH 091/128] try to fix test code --- tests/app/dao/test_service_sms_sender_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/dao/test_service_sms_sender_dao.py b/tests/app/dao/test_service_sms_sender_dao.py index bd0bb8801..b468d47f9 100644 --- a/tests/app/dao/test_service_sms_sender_dao.py +++ b/tests/app/dao/test_service_sms_sender_dao.py @@ -4,6 +4,7 @@ import pytest from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from app import db from app.dao.service_sms_sender_dao import ( archive_sms_sender, dao_add_sms_sender_for_service, @@ -14,7 +15,6 @@ from app.dao.service_sms_sender_dao import ( ) from app.exceptions import ArchiveValidationError from app.models import ServiceSmsSender -from tests.app import db from tests.app.db import ( create_inbound_number, create_service, From c6fee4a0b04cfbee8a55a52d026da1f5f1708dd9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 08:47:23 -0700 Subject: [PATCH 092/128] try to fix test code --- tests/app/dao/test_service_sms_sender_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/dao/test_service_sms_sender_dao.py b/tests/app/dao/test_service_sms_sender_dao.py index b468d47f9..10bfd21f4 100644 --- a/tests/app/dao/test_service_sms_sender_dao.py +++ b/tests/app/dao/test_service_sms_sender_dao.py @@ -138,7 +138,7 @@ def test_dao_update_service_sms_sender(notify_db_session): sms_sender="updated", ) stmt = select(ServiceSmsSender).filter_by(service_id=service.id) - sms_senders = db.sessions.execute(stmt).scalars().all() + sms_senders = db.session.execute(stmt).scalars().all() assert len(sms_senders) == 1 assert sms_senders[0].is_default assert sms_senders[0].sms_sender == "updated" From b5f7977a481cce794533d2ce834666638b84813b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 09:16:19 -0700 Subject: [PATCH 093/128] fix core daos --- app/dao/notifications_dao.py | 106 +++++++++++++++++++++-------------- app/service_invite/rest.py | 2 +- 2 files changed, 65 insertions(+), 43 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index f7150d08f..360c6af35 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,7 +1,7 @@ from datetime import timedelta from flask import current_app -from sqlalchemy import asc, desc, or_, select, text, union +from sqlalchemy import asc, delete, desc, func, or_, select, text, union, update from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.sql import functions @@ -109,11 +109,12 @@ def _update_notification_status( def update_notification_status_by_id( notification_id, status, sent_by=None, provider_response=None, carrier=None ): - notification = ( - Notification.query.with_for_update() + stmt = ( + select(Notification) + .with_for_update() .filter(Notification.id == notification_id) - .first() ) + notification = db.session.execute(stmt).scalars().first() if not notification: current_app.logger.info( @@ -156,9 +157,8 @@ def update_notification_status_by_id( @autocommit def update_notification_status_by_reference(reference, status): # this is used to update emails - notification = Notification.query.filter( - Notification.reference == reference - ).first() + stmt = select(Notification).filter(Notification.reference == reference) + notification = db.session.execute(stmt).scalars().first() if not notification: current_app.logger.error( @@ -200,19 +200,20 @@ def get_notifications_for_job( def dao_get_notification_count_for_job_id(*, job_id): - return Notification.query.filter_by(job_id=job_id).count() + stmt = select(func.count(Notification.id)).filter_by(job_id=job_id) + return db.session.execute(stmt).scalar() def dao_get_notification_count_for_service(*, service_id): - notification_count = Notification.query.filter_by(service_id=service_id).count() - return notification_count + stmt = select(func.count(Notification.id)).filter_by(service_id=service_id) + return db.session.execute(stmt).scalar() def dao_get_failed_notification_count(): - failed_count = Notification.query.filter_by( + stmt = select(func.count(Notification.id)).filter_by( status=NotificationStatus.FAILED - ).count() - return failed_count + ) + return db.session.execute(stmt).scalar() def get_notification_with_personalisation(service_id, notification_id, key_type): @@ -220,11 +221,12 @@ def get_notification_with_personalisation(service_id, notification_id, key_type) if key_type: filter_dict["key_type"] = key_type - return ( - Notification.query.filter_by(**filter_dict) + stmt = ( + select(Notification) + .filter_by(**filter_dict) .options(joinedload(Notification.template)) - .one() ) + return db.session.execute(stmt).scalars().one() def get_notification_by_id(notification_id, service_id=None, _raise=False): @@ -233,9 +235,13 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False): if service_id: filters.append(Notification.service_id == service_id) - query = Notification.query.filter(*filters) + stmt = select(Notification).filter(*filters) - return query.one() if _raise else query.first() + return ( + db.session.execute(stmt).scalars().one() + if _raise + else db.session.execute(stmt).scalars().first() + ) def get_notifications_for_service( @@ -415,12 +421,13 @@ def move_notifications_to_notification_history( deleted += delete_count_per_call # Deleting test Notifications, test notifications are not persisted to NotificationHistory - Notification.query.filter( + stmt = delete(Notification).filter( Notification.notification_type == notification_type, Notification.service_id == service_id, Notification.created_at < timestamp_to_delete_backwards_from, Notification.key_type == KeyType.TEST, - ).delete(synchronize_session=False) + ) + db.session.execute(stmt) db.session.commit() return deleted @@ -442,8 +449,9 @@ def dao_timeout_notifications(cutoff_time, limit=100000): current_statuses = [NotificationStatus.SENDING, NotificationStatus.PENDING] new_status = NotificationStatus.TEMPORARY_FAILURE - notifications = ( - Notification.query.filter( + stmt = ( + select(Notification) + .filter( Notification.created_at < cutoff_time, Notification.status.in_(current_statuses), Notification.notification_type.in_( @@ -451,14 +459,17 @@ def dao_timeout_notifications(cutoff_time, limit=100000): ), ) .limit(limit) - .all() ) + notifications = db.session.execute(stmt).scalars().all() - Notification.query.filter( - Notification.id.in_([n.id for n in notifications]), - ).update( - {"status": new_status, "updated_at": updated_at}, synchronize_session=False + stmt = ( + update(Notification) + .filter(Notification.id.in_([n.id for n in notifications])) + .update( + {"status": new_status, "updated_at": updated_at}, synchronize_session=False + ) ) + db.session.execute(stmt) db.session.commit() return notifications @@ -466,15 +477,21 @@ def dao_timeout_notifications(cutoff_time, limit=100000): @autocommit def dao_update_notifications_by_reference(references, update_dict): - updated_count = Notification.query.filter( - Notification.reference.in_(references) - ).update(update_dict, synchronize_session=False) + stmt = ( + update(Notification) + .filter(Notification.reference.in_(references)) + .update(update_dict) + ) + updated_count = db.stmt.execute(stmt) updated_history_count = 0 if updated_count != len(references): - updated_history_count = NotificationHistory.query.filter( - NotificationHistory.reference.in_(references) - ).update(update_dict, synchronize_session=False) + stmt = ( + select(NotificationHistory) + .filter(NotificationHistory.reference.in_(references)) + .update(update_dict, synchronize_session=False) + ) + updated_history_count = db.stmt.execute(stmt) return updated_count, updated_history_count @@ -541,18 +558,21 @@ def dao_get_notifications_by_recipient_or_reference( def dao_get_notification_by_reference(reference): - return Notification.query.filter(Notification.reference == reference).one() + stmt = select(Notification).filter(Notification.reference == reference) + return db.session.execute(stmt).scalars().one() def dao_get_notification_history_by_reference(reference): try: # This try except is necessary because in test keys and research mode does not create notification history. # Otherwise we could just search for the NotificationHistory object - return Notification.query.filter(Notification.reference == reference).one() + stmt = select(Notification).filter(Notification.reference == reference) + return db.session.execute(stmt).scalars().one() except NoResultFound: - return NotificationHistory.query.filter( + stmt = select(NotificationHistory).filter( NotificationHistory.reference == reference - ).one() + ) + return db.session.execute(stmt).scalars().one() def dao_get_notifications_processing_time_stats(start_date, end_date): @@ -590,11 +610,12 @@ def dao_get_notifications_processing_time_stats(start_date, end_date): def dao_get_last_notification_added_for_job_id(job_id): - last_notification_added = ( - Notification.query.filter(Notification.job_id == job_id) + stmt = ( + select(Notification) + .filter(Notification.job_id == job_id) .order_by(Notification.job_row_number.desc()) - .first() ) + last_notification_added = db.session.execute(stmt).scalars().first() return last_notification_added @@ -602,11 +623,12 @@ def dao_get_last_notification_added_for_job_id(job_id): def notifications_not_yet_sent(should_be_sending_after_seconds, notification_type): older_than_date = utc_now() - timedelta(seconds=should_be_sending_after_seconds) - notifications = Notification.query.filter( + stmt = select(Notification).filter( Notification.created_at <= older_than_date, Notification.notification_type == notification_type, Notification.status == NotificationStatus.CREATED, - ).all() + ) + notifications = db.session.execute(stmt).all() return notifications diff --git a/app/service_invite/rest.py b/app/service_invite/rest.py index f6d9627da..5728b3ed5 100644 --- a/app/service_invite/rest.py +++ b/app/service_invite/rest.py @@ -86,7 +86,7 @@ def _create_service_invite(invited_user, invite_link_host): redis_store.set( f"email-personalisation-{saved_notification.id}", json.dumps(personalisation), - ex=2*24*60*60, + ex=2 * 24 * 60 * 60, ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) From 88c9af90505cf471e98b496dbd8ec2606261875e Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 09:31:47 -0700 Subject: [PATCH 094/128] fix core daos --- app/dao/notifications_dao.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 360c6af35..4d1fee9d1 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -465,7 +465,7 @@ def dao_timeout_notifications(cutoff_time, limit=100000): stmt = ( update(Notification) .filter(Notification.id.in_([n.id for n in notifications])) - .update( + .values( {"status": new_status, "updated_at": updated_at}, synchronize_session=False ) ) @@ -480,7 +480,7 @@ def dao_update_notifications_by_reference(references, update_dict): stmt = ( update(Notification) .filter(Notification.reference.in_(references)) - .update(update_dict) + .values(update_dict) ) updated_count = db.stmt.execute(stmt) @@ -489,7 +489,7 @@ def dao_update_notifications_by_reference(references, update_dict): stmt = ( select(NotificationHistory) .filter(NotificationHistory.reference.in_(references)) - .update(update_dict, synchronize_session=False) + .values(update_dict, synchronize_session=False) ) updated_history_count = db.stmt.execute(stmt) From 90b407241f971885d5d4c0c3aa9c4c9c001ef967 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 09:38:12 -0700 Subject: [PATCH 095/128] fix core daos --- app/dao/notifications_dao.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 4d1fee9d1..ca1646467 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -466,7 +466,7 @@ def dao_timeout_notifications(cutoff_time, limit=100000): update(Notification) .filter(Notification.id.in_([n.id for n in notifications])) .values( - {"status": new_status, "updated_at": updated_at}, synchronize_session=False + {"status": new_status, "updated_at": updated_at} ) ) db.session.execute(stmt) @@ -482,16 +482,16 @@ def dao_update_notifications_by_reference(references, update_dict): .filter(Notification.reference.in_(references)) .values(update_dict) ) - updated_count = db.stmt.execute(stmt) + updated_count = db.session.execute(stmt) updated_history_count = 0 if updated_count != len(references): stmt = ( select(NotificationHistory) .filter(NotificationHistory.reference.in_(references)) - .values(update_dict, synchronize_session=False) + .values(update_dict) ) - updated_history_count = db.stmt.execute(stmt) + updated_history_count = db.session.execute(stmt) return updated_count, updated_history_count From d8bb71bf39eaae872d8d1cc195de7b9f376115e9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 09:47:00 -0700 Subject: [PATCH 096/128] fix core daos --- app/dao/notifications_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index ca1646467..d1c0d7202 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -487,7 +487,7 @@ def dao_update_notifications_by_reference(references, update_dict): updated_history_count = 0 if updated_count != len(references): stmt = ( - select(NotificationHistory) + update(NotificationHistory) .filter(NotificationHistory.reference.in_(references)) .values(update_dict) ) From 958861df59503eac58b6f75bc32e30b9f91bae8c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 09:59:16 -0700 Subject: [PATCH 097/128] fix core daos --- app/dao/notifications_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index d1c0d7202..1fbb637c9 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -482,7 +482,7 @@ def dao_update_notifications_by_reference(references, update_dict): .filter(Notification.reference.in_(references)) .values(update_dict) ) - updated_count = db.session.execute(stmt) + updated_count = db.session.execute(stmt).scalar() or 0 updated_history_count = 0 if updated_count != len(references): @@ -491,7 +491,7 @@ def dao_update_notifications_by_reference(references, update_dict): .filter(NotificationHistory.reference.in_(references)) .values(update_dict) ) - updated_history_count = db.session.execute(stmt) + updated_history_count = db.session.execute(stmt).scalar() or 0 return updated_count, updated_history_count From c44300a73707cf9d6e62c127a23bf8498b4b8e46 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 10:13:11 -0700 Subject: [PATCH 098/128] fix core daos --- app/dao/notifications_dao.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 1fbb637c9..802c2a287 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -465,9 +465,7 @@ def dao_timeout_notifications(cutoff_time, limit=100000): stmt = ( update(Notification) .filter(Notification.id.in_([n.id for n in notifications])) - .values( - {"status": new_status, "updated_at": updated_at} - ) + .values({"status": new_status, "updated_at": updated_at}) ) db.session.execute(stmt) @@ -482,7 +480,8 @@ def dao_update_notifications_by_reference(references, update_dict): .filter(Notification.reference.in_(references)) .values(update_dict) ) - updated_count = db.session.execute(stmt).scalar() or 0 + result = db.session.execute(stmt) + updated_count = result.rowcount updated_history_count = 0 if updated_count != len(references): @@ -491,7 +490,8 @@ def dao_update_notifications_by_reference(references, update_dict): .filter(NotificationHistory.reference.in_(references)) .values(update_dict) ) - updated_history_count = db.session.execute(stmt).scalar() or 0 + result = db.session.execute(stmt) + updated_history_count = result.rowcount return updated_count, updated_history_count From 84e46f1ef2e25fb9265e5ed5f199f9927271d46f Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 10:29:13 -0700 Subject: [PATCH 099/128] fix core daos --- app/dao/notifications_dao.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 802c2a287..e93f59e28 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -629,6 +629,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ Notification.status == NotificationStatus.CREATED, ) notifications = db.session.execute(stmt).all() + print(f"WE RETURN THIS FOR NOTIFICATIONS {notifications}") return notifications From ef6e4048c27368c7dd5914203cbcfe47e84919d5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 10:38:28 -0700 Subject: [PATCH 100/128] fix core daos --- app/dao/notifications_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index e93f59e28..8659fca9b 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -628,7 +628,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ Notification.notification_type == notification_type, Notification.status == NotificationStatus.CREATED, ) - notifications = db.session.execute(stmt).all() + notifications = db.session.execute(stmt).scalars().all() print(f"WE RETURN THIS FOR NOTIFICATIONS {notifications}") return notifications From 4f20bfe2dbc1ba16dc75bf5f52769aad8a9f17e7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 11:30:35 -0700 Subject: [PATCH 101/128] fix core daos --- app/dao/users_dao.py | 55 ++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index 897bb1b9e..07995ac6a 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -4,7 +4,7 @@ from secrets import randbelow import sqlalchemy from flask import current_app -from sqlalchemy import func, text +from sqlalchemy import delete, func, select, text from sqlalchemy.orm import joinedload from app import db @@ -37,8 +37,8 @@ def get_login_gov_user(login_uuid, email_address): login.gov uuids are. Eventually the code that checks by email address should be removed. """ - - user = User.query.filter_by(login_uuid=login_uuid).first() + stmt = select(User).filter_by(login_uuid=login_uuid) + user = db.session.execute(stmt).scalars().first() if user: if user.email_address != email_address: try: @@ -54,7 +54,8 @@ def get_login_gov_user(login_uuid, email_address): return user # Remove this 1 July 2025, all users should have login.gov uuids by now - user = User.query.filter(User.email_address.ilike(email_address)).first() + stmt = select(User).filter(User.email_address.ilike(email_address)) + user = db.session.execute(stmt).scalars().first() if user: save_user_attribute(user, {"login_uuid": login_uuid}) @@ -102,24 +103,27 @@ def create_user_code(user, code, code_type): def get_user_code(user, code, code_type): # Get the most recent codes to try and reduce the # time searching for the correct code. - codes = VerifyCode.query.filter_by(user=user, code_type=code_type).order_by( - VerifyCode.created_at.desc() + stmt = ( + select(VerifyCode) + .filter_by(user=user, code_type=code_type) + .order_by(VerifyCode.created_at.desc()) ) + codes = db.session.execute(stmt).scalars().all() return next((x for x in codes if x.check_code(code)), None) def delete_codes_older_created_more_than_a_day_ago(): - deleted = ( - db.session.query(VerifyCode) - .filter(VerifyCode.created_at < utc_now() - timedelta(hours=24)) - .delete() + stmt = delete(VerifyCode).filter( + VerifyCode.created_at < utc_now() - timedelta(hours=24) ) + + deleted = db.session.execute(stmt) db.session.commit() return deleted def use_user_code(id): - verify_code = VerifyCode.query.get(id) + verify_code = db.session.get(VerifyCode, id) verify_code.code_used = True db.session.add(verify_code) db.session.commit() @@ -131,36 +135,42 @@ def delete_model_user(user): def delete_user_verify_codes(user): - VerifyCode.query.filter_by(user=user).delete() + stmt = delete(VerifyCode).filter_by(user=user) + db.session.execute(stmt) db.session.commit() def count_user_verify_codes(user): - query = VerifyCode.query.filter( + stmt = select(func.count(VerifyCode.id)).filter( VerifyCode.user == user, VerifyCode.expiry_datetime > utc_now(), VerifyCode.code_used.is_(False), ) - return query.count() + result = db.session.execute(stmt) + return result.rowcount def get_user_by_id(user_id=None): if user_id: - return User.query.filter_by(id=user_id).one() - return User.query.filter_by().all() + stmt = select(User).filter_by(id=user_id) + return db.session.execute(stmt).scalars().one() + return get_users() def get_users(): - return User.query.all() + stmt = select(User) + return db.session.execute(stmt).scalars().all() def get_user_by_email(email): - return User.query.filter(func.lower(User.email_address) == func.lower(email)).one() + stmt = select(User).filter(func.lower(User.email_address) == func.lower(email)) + return db.session.execute(stmt).scalars().one() def get_users_by_partial_email(email): email = escape_special_characters(email) - return User.query.filter(User.email_address.ilike("%{}%".format(email))).all() + stmt = select(User).filter(User.email_address.ilike("%{}%".format(email))) + return db.session.execute(stmt).scalars().all() def increment_failed_login_count(user): @@ -188,16 +198,17 @@ def get_user_and_accounts(user_id): # TODO: With sqlalchemy 2.0 change as below because of the breaking change # at User.organizations.services, we need to verify that the below subqueryload # that we have put is functionally doing the same thing as before - return ( - User.query.filter(User.id == user_id) + stmt = ( + select(User) + .filter(User.id == user_id) .options( # eagerly load the user's services and organizations, and also the service's org and vice versa # (so we can see if the user knows about it) joinedload(User.services).joinedload(Service.organization), joinedload(User.organizations).subqueryload(Organization.services), ) - .one() ) + return db.session.execute(stmt).scalars().one() @autocommit From a5eceae07b3d1f0a2e50300e21f4256143366d33 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 11:39:48 -0700 Subject: [PATCH 102/128] fix core daos --- app/dao/users_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index 07995ac6a..b0b5f2679 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -146,8 +146,8 @@ def count_user_verify_codes(user): VerifyCode.expiry_datetime > utc_now(), VerifyCode.code_used.is_(False), ) - result = db.session.execute(stmt) - return result.rowcount + result = db.session.execute(stmt).scalar() + return result or 0 def get_user_by_id(user_id=None): From 1f6decebe2876d081bb0c066d5c2693d07ba48b1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 12:00:04 -0700 Subject: [PATCH 103/128] fix core daos --- app/dao/users_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index b0b5f2679..690ecc7f9 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -208,7 +208,7 @@ def get_user_and_accounts(user_id): joinedload(User.organizations).subqueryload(Organization.services), ) ) - return db.session.execute(stmt).scalars().one() + return db.session.execute(stmt).scalars().unique().one() @autocommit From 54ab96e0737fb1bd5b223458e4d9b0d4cffd9833 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 12:26:55 -0700 Subject: [PATCH 104/128] fix core daos --- .ds.baseline | 6 +- .../notification_dao/test_notification_dao.py | 234 +++++++++++------- tests/app/dao/test_users_dao.py | 35 ++- 3 files changed, 176 insertions(+), 99 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 1c279e018..37199f01f 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -249,7 +249,7 @@ "filename": "tests/app/dao/test_users_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 52, + "line_number": 69, "is_secret": false }, { @@ -257,7 +257,7 @@ "filename": "tests/app/dao/test_users_dao.py", "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba", "is_verified": false, - "line_number": 176, + "line_number": 194, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-09-27T16:42:53Z" + "generated_at": "2024-10-11T19:26:50Z" } diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 4bc1ce5ba..8e81db3a2 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -4,9 +4,11 @@ from functools import partial import pytest from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError, SQLAlchemyError from sqlalchemy.orm.exc import NoResultFound +from app import db from app.dao.notifications_dao import ( dao_create_notification, dao_delete_notifications_by_id, @@ -55,7 +57,10 @@ def test_should_by_able_to_update_status_by_reference( notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.get(notification.id).status == NotificationStatus.SENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.SENDING + ) notification.reference = "reference" dao_update_notification(notification) @@ -64,7 +69,8 @@ def test_should_by_able_to_update_status_by_reference( ) assert updated.status == NotificationStatus.DELIVERED assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) @@ -81,7 +87,10 @@ def test_should_by_able_to_update_status_by_id( dao_create_notification(notification) assert notification.status == NotificationStatus.SENDING - assert Notification.query.get(notification.id).status == NotificationStatus.SENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.SENDING + ) with freeze_time("2000-01-02 12:00:00"): updated = update_notification_status_by_id( @@ -92,7 +101,8 @@ def test_should_by_able_to_update_status_by_id( assert updated.status == NotificationStatus.DELIVERED assert updated.updated_at == datetime(2000, 1, 2, 12, 0, 0) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) assert notification.updated_at == datetime(2000, 1, 2, 12, 0, 0) assert notification.status == NotificationStatus.DELIVERED @@ -107,15 +117,17 @@ def test_should_not_update_status_by_id_if_not_sending_and_does_not_update_job( job=sample_job, ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) assert not update_notification_status_by_id( notification.id, NotificationStatus.FAILED ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) - assert sample_job == Job.query.get(notification.job_id) + assert sample_job == db.session.get(Job, notification.job_id) def test_should_not_update_status_by_reference_if_not_sending_and_does_not_update_job( @@ -128,20 +140,22 @@ def test_should_not_update_status_by_reference_if_not_sending_and_does_not_updat job=sample_job, ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) assert not update_notification_status_by_reference( "reference", NotificationStatus.FAILED ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) - assert sample_job == Job.query.get(notification.job_id) + assert sample_job == db.session.get(Job, notification.job_id) def test_should_update_status_by_id_if_created(sample_template, sample_notification): assert ( - Notification.query.get(sample_notification.id).status + db.session.get(Notification, sample_notification.id).status == NotificationStatus.CREATED ) updated = update_notification_status_by_id( @@ -149,7 +163,7 @@ def test_should_update_status_by_id_if_created(sample_template, sample_notificat NotificationStatus.FAILED, ) assert ( - Notification.query.get(sample_notification.id).status + db.session.get(Notification, sample_notification.id).status == NotificationStatus.FAILED ) assert updated.status == NotificationStatus.FAILED @@ -244,11 +258,17 @@ def test_should_not_update_status_by_reference_if_not_sending(sample_template): status=NotificationStatus.CREATED, reference="reference", ) - assert Notification.query.get(notification.id).status == NotificationStatus.CREATED + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.CREATED + ) updated = update_notification_status_by_reference( "reference", NotificationStatus.FAILED ) - assert Notification.query.get(notification.id).status == NotificationStatus.CREATED + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.CREATED + ) assert not updated @@ -264,14 +284,18 @@ def test_should_by_able_to_update_status_by_id_from_pending_to_delivered( assert update_notification_status_by_id( notification_id=notification.id, status=NotificationStatus.PENDING ) - assert Notification.query.get(notification.id).status == NotificationStatus.PENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.PENDING + ) assert update_notification_status_by_id( notification.id, NotificationStatus.DELIVERED, ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) @@ -289,7 +313,10 @@ def test_should_by_able_to_update_status_by_id_from_pending_to_temporary_failure notification_id=notification.id, status=NotificationStatus.PENDING, ) - assert Notification.query.get(notification.id).status == NotificationStatus.PENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.PENDING + ) assert update_notification_status_by_id( notification.id, @@ -297,7 +324,7 @@ def test_should_by_able_to_update_status_by_id_from_pending_to_temporary_failure ) assert ( - Notification.query.get(notification.id).status + db.session.get(Notification, notification.id).status == NotificationStatus.TEMPORARY_FAILURE ) @@ -312,14 +339,17 @@ def test_should_by_able_to_update_status_by_id_from_sending_to_permanent_failure ) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.get(notification.id).status == NotificationStatus.SENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.SENDING + ) assert update_notification_status_by_id( notification.id, status=NotificationStatus.PERMANENT_FAILURE, ) assert ( - Notification.query.get(notification.id).status + db.session.get(Notification, notification.id).status == NotificationStatus.PERMANENT_FAILURE ) @@ -331,7 +361,10 @@ def test_should_not_update_status_once_notification_status_is_delivered( template=sample_email_template, status=NotificationStatus.SENDING, ) - assert Notification.query.get(notification.id).status == NotificationStatus.SENDING + assert ( + db.session.get(Notification, notification.id).status + == NotificationStatus.SENDING + ) notification.reference = "reference" dao_update_notification(notification) @@ -340,7 +373,8 @@ def test_should_not_update_status_once_notification_status_is_delivered( NotificationStatus.DELIVERED, ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) update_notification_status_by_reference( @@ -348,7 +382,8 @@ def test_should_not_update_status_once_notification_status_is_delivered( NotificationStatus.FAILED, ) assert ( - Notification.query.get(notification.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification.id).status + == NotificationStatus.DELIVERED ) @@ -370,7 +405,7 @@ def test_create_notification_creates_notification_with_personalisation( sample_template_with_placeholders, sample_job, ): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = create_notification( template=sample_template_with_placeholders, @@ -379,8 +414,8 @@ def test_create_notification_creates_notification_with_personalisation( status=NotificationStatus.CREATED, ) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert data.to == notification_from_db.to assert data.job_id == notification_from_db.job_id @@ -393,15 +428,15 @@ def test_create_notification_creates_notification_with_personalisation( def test_save_notification_creates_sms(sample_template, sample_job): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template, job_id=sample_job.id) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["job_id"] == notification_from_db.job_id @@ -412,16 +447,36 @@ def test_save_notification_creates_sms(sample_template, sample_job): assert notification_from_db.status == NotificationStatus.CREATED +def _get_notification_query_all(): + stmt = select(Notification) + return db.execute(stmt).scalars().all() + + +def _get_notification_query_one(): + stmt = select(Notification) + return db.execute(stmt).scalars().one() + + +def _get_notification_query_count(): + stmt = select(func.count(Notification.id)) + return db.session.execute(stmt).scalar() or 0 + + +def _get_notification_history_query_count(): + stmt = select(func.count(NotificationHistory.id)) + return db.session.execute(stmt).scalar() or 0 + + def test_save_notification_and_create_email(sample_email_template, sample_job): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_email_template, job_id=sample_job.id) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["job_id"] == notification_from_db.job_id @@ -433,29 +488,29 @@ def test_save_notification_and_create_email(sample_email_template, sample_job): def test_save_notification(sample_email_template, sample_job): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_email_template, job_id=sample_job.id) notification_1 = Notification(**data) notification_2 = Notification(**data) dao_create_notification(notification_1) - assert Notification.query.count() == 1 + assert _get_notification_query_count() == 1 dao_create_notification(notification_2) - assert Notification.query.count() == 2 + assert _get_notification_query_count() == 2 def test_save_notification_does_not_creates_history(sample_email_template, sample_job): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_email_template, job_id=sample_job.id) notification_1 = Notification(**data) dao_create_notification(notification_1) - assert Notification.query.count() == 1 - assert NotificationHistory.query.count() == 0 + assert _get_notification_query_count() == 1 + assert _get_notification_history_query_count() == 0 def test_update_notification_with_research_mode_service_does_not_create_or_update_history( @@ -464,14 +519,14 @@ def test_update_notification_with_research_mode_service_does_not_create_or_updat sample_template.service.research_mode = True notification = create_notification(template=sample_template) - assert Notification.query.count() == 1 - assert NotificationHistory.query.count() == 0 + assert _get_notification_query_count() == 1 + assert _get_notification_history_query_count() == 0 notification.status = NotificationStatus.DELIVERED dao_update_notification(notification) - assert Notification.query.one().status == NotificationStatus.DELIVERED - assert NotificationHistory.query.count() == 0 + assert _get_notification_query_one().status == NotificationStatus.DELIVERED + assert _get_notification_history_query_count() == 0 def test_not_save_notification_and_not_create_stats_on_commit_error( @@ -479,26 +534,26 @@ def test_not_save_notification_and_not_create_stats_on_commit_error( ): random_id = str(uuid.uuid4()) - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template, job_id=random_id) notification = Notification(**data) with pytest.raises(SQLAlchemyError): dao_create_notification(notification) - assert Notification.query.count() == 0 - assert Job.query.get(sample_job.id).notifications_sent == 0 + assert _get_notification_query_count() == 0 + assert db.session.get(Job, sample_job.id).notifications_sent == 0 def test_save_notification_and_increment_job(sample_template, sample_job, sns_provider): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template, job_id=sample_job.id) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["job_id"] == notification_from_db.job_id @@ -510,21 +565,21 @@ def test_save_notification_and_increment_job(sample_template, sample_job, sns_pr notification_2 = Notification(**data) dao_create_notification(notification_2) - assert Notification.query.count() == 2 + assert _get_notification_query_count() == 2 def test_save_notification_and_increment_correct_job(sample_template, sns_provider): job_1 = create_job(sample_template) job_2 = create_job(sample_template) - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template, job_id=job_1.id) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["job_id"] == notification_from_db.job_id @@ -537,14 +592,14 @@ def test_save_notification_and_increment_correct_job(sample_template, sns_provid def test_save_notification_with_no_job(sample_template, sns_provider): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["service"] == notification_from_db.service @@ -592,7 +647,7 @@ def test_get_notification_by_id_when_notification_exists_for_different_service( def test_get_notifications_by_reference(sample_template): client_reference = "some-client-ref" - assert len(Notification.query.all()) == 0 + assert len(_get_notification_query_all()) == 0 create_notification(sample_template, client_reference=client_reference) create_notification(sample_template, client_reference=client_reference) create_notification(sample_template, client_reference="other-ref") @@ -603,14 +658,14 @@ def test_get_notifications_by_reference(sample_template): def test_save_notification_no_job_id(sample_template): - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 data = _notification_json(sample_template) notification = Notification(**data) dao_create_notification(notification) - assert Notification.query.count() == 1 - notification_from_db = Notification.query.all()[0] + assert _get_notification_query_count() == 1 + notification_from_db = _get_notification_query_all()[0] assert notification_from_db.id assert "1" == notification_from_db.to assert data["service"] == notification_from_db.service @@ -687,13 +742,13 @@ def test_update_notification_sets_status(sample_notification): assert sample_notification.status == NotificationStatus.CREATED sample_notification.status = NotificationStatus.FAILED dao_update_notification(sample_notification) - notification_from_db = Notification.query.get(sample_notification.id) + notification_from_db = db.session.get(Notification, sample_notification.id) assert notification_from_db.status == NotificationStatus.FAILED @freeze_time("2016-01-10") def test_should_limit_notifications_return_by_day_limit_plus_one(sample_template): - assert len(Notification.query.all()) == 0 + assert len(_get_notification_query_all()) == 0 # create one notification a day between 1st and 9th, # with assumption that the local timezone is EST @@ -706,7 +761,7 @@ def test_should_limit_notifications_return_by_day_limit_plus_one(sample_template status=NotificationStatus.FAILED, ) - all_notifications = Notification.query.all() + all_notifications = _get_notification_query_all() assert len(all_notifications) == 10 all_notifications = get_notifications_for_service( @@ -722,19 +777,19 @@ def test_should_limit_notifications_return_by_day_limit_plus_one(sample_template def test_creating_notification_does_not_add_notification_history(sample_template): create_notification(template=sample_template) - assert Notification.query.count() == 1 - assert NotificationHistory.query.count() == 0 + assert _get_notification_query_count() == 1 + assert _get_notification_history_query_count() == 0 def test_should_delete_notification_for_id(sample_template): notification = create_notification(template=sample_template) - assert Notification.query.count() == 1 - assert NotificationHistory.query.count() == 0 + assert _get_notification_query_count() == 1 + assert _get_notification_history_query_count() == 0 dao_delete_notifications_by_id(notification.id) - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 def test_should_delete_notification_and_ignore_history_for_research_mode( @@ -744,31 +799,32 @@ def test_should_delete_notification_and_ignore_history_for_research_mode( notification = create_notification(template=sample_template) - assert Notification.query.count() == 1 + assert _get_notification_query_count() == 1 dao_delete_notifications_by_id(notification.id) - assert Notification.query.count() == 0 + assert _get_notification_query_count() == 0 def test_should_delete_only_notification_with_id(sample_template): notification_1 = create_notification(template=sample_template) notification_2 = create_notification(template=sample_template) - assert Notification.query.count() == 2 + assert _get_notification_query_count() == 2 dao_delete_notifications_by_id(notification_1.id) - assert Notification.query.count() == 1 - assert Notification.query.first().id == notification_2.id + assert _get_notification_query_count() == 1 + stmt = select(Notification) + assert db.session.execute(stmt).scalars().first().id == notification_2.id def test_should_delete_no_notifications_if_no_matching_ids(sample_template): create_notification(template=sample_template) - assert Notification.query.count() == 1 + assert _get_notification_query_count() == 1 dao_delete_notifications_by_id(uuid.uuid4()) - assert Notification.query.count() == 1 + assert _get_notification_query_count() == 1 def _notification_json(sample_template, job_id=None, id=None, status=None): @@ -814,16 +870,19 @@ def test_dao_timeout_notifications(sample_template): temporary_failure_notifications = dao_timeout_notifications(utc_now()) assert len(temporary_failure_notifications) == 2 - assert Notification.query.get(created.id).status == NotificationStatus.CREATED + assert db.session.get(Notification, created.id).status == NotificationStatus.CREATED assert ( - Notification.query.get(sending.id).status + db.session.get(Notification, sending.id).status == NotificationStatus.TEMPORARY_FAILURE ) assert ( - Notification.query.get(pending.id).status + db.session.get(Notification, pending.id).status == NotificationStatus.TEMPORARY_FAILURE ) - assert Notification.query.get(delivered.id).status == NotificationStatus.DELIVERED + assert ( + db.session.get(Notification, delivered.id).status + == NotificationStatus.DELIVERED + ) def test_dao_timeout_notifications_only_updates_for_older_notifications( @@ -842,8 +901,8 @@ def test_dao_timeout_notifications_only_updates_for_older_notifications( temporary_failure_notifications = dao_timeout_notifications(utc_now()) assert len(temporary_failure_notifications) == 0 - assert Notification.query.get(sending.id).status == NotificationStatus.SENDING - assert Notification.query.get(pending.id).status == NotificationStatus.PENDING + assert db.session.get(Notification, sending.id).status == NotificationStatus.SENDING + assert db.session.get(Notification, pending.id).status == NotificationStatus.PENDING def test_should_return_notifications_excluding_jobs_by_default( @@ -935,7 +994,7 @@ def test_get_notifications_created_by_api_or_csv_are_returned_correctly_excludin key_type=sample_test_api_key.key_type, ) - all_notifications = Notification.query.all() + all_notifications = _get_notification_query_all() assert len(all_notifications) == 4 # returns all real API derived notifications @@ -982,7 +1041,7 @@ def test_get_notifications_with_a_live_api_key_type( key_type=sample_test_api_key.key_type, ) - all_notifications = Notification.query.all() + all_notifications = _get_notification_query_all() assert len(all_notifications) == 4 # only those created with normal API key, no jobs @@ -1114,7 +1173,7 @@ def test_should_exclude_test_key_notifications_by_default( key_type=sample_test_api_key.key_type, ) - all_notifications = Notification.query.all() + all_notifications = _get_notification_query_all() assert len(all_notifications) == 4 all_notifications = get_notifications_for_service( @@ -1757,10 +1816,10 @@ def test_dao_update_notifications_by_reference_updated_notifications(sample_temp update_dict={"status": NotificationStatus.DELIVERED, "billable_units": 2}, ) assert updated_count == 2 - updated_1 = Notification.query.get(notification_1.id) + updated_1 = db.session.get(Notification, notification_1.id) assert updated_1.billable_units == 2 assert updated_1.status == NotificationStatus.DELIVERED - updated_2 = Notification.query.get(notification_2.id) + updated_2 = db.session.get(Notification, notification_2.id) assert updated_2.billable_units == 2 assert updated_2.status == NotificationStatus.DELIVERED @@ -1823,10 +1882,11 @@ def test_dao_update_notifications_by_reference_updates_history_when_one_of_two_n assert updated_count == 1 assert updated_history_count == 1 assert ( - Notification.query.get(notification2.id).status == NotificationStatus.DELIVERED + db.session.get(Notification, notification2.id).status + == NotificationStatus.DELIVERED ) assert ( - NotificationHistory.query.get(notification1.id).status + db.session.get(NotificationHistory, notification1.id).status == NotificationStatus.DELIVERED ) diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index 9c8770913..85149b246 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -3,6 +3,7 @@ from datetime import timedelta import pytest from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.exc import DataError from sqlalchemy.orm.exc import NoResultFound @@ -37,6 +38,21 @@ from tests.app.db import ( ) +def _get_user_query_count(): + stmt = select(func.count(User.id)) + return db.session.execute(stmt).scalar() or 0 + + +def _get_user_query_first(): + stmt = select(User) + return db.session.execute(stmt).scalars().first() + + +def _get_verify_code_query_count(): + stmt = select(func.count(VerifyCode.id)) + return db.session.execute(stmt).scalar() or 0 + + @freeze_time("2020-01-28T12:00:00") @pytest.mark.parametrize( "phone_number, expected_phone_number", @@ -55,8 +71,8 @@ def test_create_user(notify_db_session, phone_number, expected_phone_number): } user = User(**data) save_model_user(user, password="password", validated_email_access=True) - assert User.query.count() == 1 - user_query = User.query.first() + assert _get_user_query_count() == 1 + user_query = _get_user_query_first() assert user_query.email_address == email assert user_query.id == user.id assert user_query.mobile_number == expected_phone_number @@ -68,7 +84,7 @@ def test_get_all_users(notify_db_session): create_user(email="1@test.com") create_user(email="2@test.com") - assert User.query.count() == 2 + assert _get_user_query_count() == 2 assert len(get_user_by_id()) == 2 @@ -89,9 +105,9 @@ def test_get_user_invalid_id(notify_db_session): def test_delete_users(sample_user): - assert User.query.count() == 1 + assert _get_user_query_count() == 1 delete_model_user(sample_user) - assert User.query.count() == 0 + assert _get_user_query_count() == 0 def test_increment_failed_login_should_increment_failed_logins(sample_user): @@ -127,9 +143,9 @@ def test_get_user_by_email_is_case_insensitive(sample_user): def test_should_delete_all_verification_codes_more_than_one_day_old(sample_user): make_verify_code(sample_user, age=timedelta(hours=24), code="54321") make_verify_code(sample_user, age=timedelta(hours=24), code="54321") - assert VerifyCode.query.count() == 2 + assert _get_verify_code_query_count() == 2 delete_codes_older_created_more_than_a_day_ago() - assert VerifyCode.query.count() == 0 + assert _get_verify_code_query_count() == 0 def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user): @@ -138,9 +154,10 @@ def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user) ) make_verify_code(sample_user, age=timedelta(hours=24), code="54321") - assert VerifyCode.query.count() == 2 + assert _get_verify_code_query_count() == 2 delete_codes_older_created_more_than_a_day_ago() - assert VerifyCode.query.one()._code == "12345" + stmt = select(VerifyCode) + assert db.session.execute(stmt).scalars().one()._code == "12345" def make_verify_code(user, age=None, expiry_age=None, code="12335", code_used=False): From b84ed9c7befc872a5063c3c0b905db3912bf5ced Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 12:33:52 -0700 Subject: [PATCH 105/128] fix core daos --- tests/app/dao/notification_dao/test_notification_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 8e81db3a2..e2ac10032 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -449,12 +449,12 @@ def test_save_notification_creates_sms(sample_template, sample_job): def _get_notification_query_all(): stmt = select(Notification) - return db.execute(stmt).scalars().all() + return db.session.execute(stmt).scalars().all() def _get_notification_query_one(): stmt = select(Notification) - return db.execute(stmt).scalars().one() + return db.session.execute(stmt).scalars().one() def _get_notification_query_count(): From 7e16eeb386a1e34f20f47ce611cb04995305ebae Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 13:30:59 -0700 Subject: [PATCH 106/128] upgrade org and template dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 12 +++++++----- app/service_invite/rest.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 9e44bcdd5..3a8a5e602 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -1,3 +1,4 @@ +from sqlalchemy import select from sqlalchemy.sql.expression import func from app import db @@ -6,14 +7,15 @@ from app.models import Domain, Organization, Service, User def dao_get_organizations(): - return Organization.query.order_by( + stmt = select(Organization).order_by( Organization.active.desc(), Organization.name.asc() - ).all() + ) + return db.session.execute(stmt).scalars().all() def dao_count_organizations_with_live_services(): - return ( - db.session.query(Organization.id) + stmt = ( + select(func.count(Organization.id)) .join(Organization.services) .filter( Service.active.is_(True), @@ -21,8 +23,8 @@ def dao_count_organizations_with_live_services(): Service.count_as_live.is_(True), ) .distinct() - .count() ) + return db.session.execute(stmt).scalar() or 0 def dao_get_organization_services(organization_id): diff --git a/app/service_invite/rest.py b/app/service_invite/rest.py index f6d9627da..5728b3ed5 100644 --- a/app/service_invite/rest.py +++ b/app/service_invite/rest.py @@ -86,7 +86,7 @@ def _create_service_invite(invited_user, invite_link_host): redis_store.set( f"email-personalisation-{saved_notification.id}", json.dumps(personalisation), - ex=2*24*60*60, + ex=2 * 24 * 60 * 60, ) send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY) From daac3bf0f34aa966ee25de907ada00a8c4a05b67 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Fri, 11 Oct 2024 14:18:35 -0700 Subject: [PATCH 107/128] upgrade org and template dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 3a8a5e602..761d1b576 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -22,9 +22,9 @@ def dao_count_organizations_with_live_services(): Service.restricted.is_(False), Service.count_as_live.is_(True), ) - .distinct() + ) - return db.session.execute(stmt).scalar() or 0 + return db.session.execute(stmt).distinct().scalar() or 0 def dao_get_organization_services(organization_id): From 88910f5718349d435b496477578d6e54a86f784c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 07:34:39 -0700 Subject: [PATCH 108/128] upgrade organization_dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 38 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 761d1b576..7fde65a2a 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -1,4 +1,4 @@ -from sqlalchemy import select +from sqlalchemy import delete, select from sqlalchemy.sql.expression import func from app import db @@ -22,41 +22,42 @@ def dao_count_organizations_with_live_services(): Service.restricted.is_(False), Service.count_as_live.is_(True), ) - ) return db.session.execute(stmt).distinct().scalar() or 0 def dao_get_organization_services(organization_id): - return Organization.query.filter_by(id=organization_id).one().services + stmt = select(Organization).filter_by(id=organization_id) + return db.session.execute(stmt).scalars().one().services def dao_get_organization_live_services(organization_id): - return Service.query.filter_by( - organization_id=organization_id, restricted=False - ).all() + stmt = select(Service).filter_by(organization_id=organization_id, restricted=False) + return db.session.execute(stmt).scalars().all() def dao_get_organization_by_id(organization_id): - return Organization.query.filter_by(id=organization_id).one() + stmt = select(Organization).filter_by(id=organization_id) + return db.session.execute(stmt).scalars().one() def dao_get_organization_by_email_address(email_address): email_address = email_address.lower().replace(".gsi.gov.uk", ".gov.uk") - - for domain in Domain.query.order_by(func.char_length(Domain.domain).desc()).all(): + stmt = select(Domain).order_by(func.char_length(Domain.domain).desc()) + domains = db.session.execute(stmt).scalars().all() + for domain in domains: if email_address.endswith( "@{}".format(domain.domain) ) or email_address.endswith(".{}".format(domain.domain)): - return Organization.query.filter_by(id=domain.organization_id).one() + stmt = select(Organization).filter_by(id=domain.organization_id) + return db.session.execute(stmt).scalars().one() return None def dao_get_organization_by_service_id(service_id): - return ( - Organization.query.join(Organization.services).filter_by(id=service_id).first() - ) + stmt = select(Organization).join(Organization.services).filter_by(id=service_id) + return db.session.execute(stmt).scalars().first() @autocommit @@ -70,7 +71,8 @@ def dao_update_organization(organization_id, **kwargs): num_updated = Organization.query.filter_by(id=organization_id).update(kwargs) if isinstance(domains, list): - Domain.query.filter_by(organization_id=organization_id).delete() + stmt = delete(Domain).filter_by(organization_id=organization_id) + db.session.execute(stmt) db.session.bulk_save_objects( [ Domain(domain=domain.lower(), organization_id=organization_id) @@ -78,7 +80,7 @@ def dao_update_organization(organization_id, **kwargs): ] ) - organization = Organization.query.get(organization_id) + organization = db.session.get(Organization, organization_id) if "organization_type" in kwargs: _update_organization_services( organization, "organization_type", only_where_none=False @@ -103,7 +105,8 @@ def _update_organization_services(organization, attribute, only_where_none=True) @autocommit @version_class(Service) def dao_add_service_to_organization(service, organization_id): - organization = Organization.query.filter_by(id=organization_id).one() + stmt = select(Organization).filter_by(id=organization_id) + organization = db.session.execute(stmt).scalars().one() service.organization_id = organization_id service.organization_type = organization.organization_type @@ -124,7 +127,8 @@ def dao_get_users_for_organization(organization_id): @autocommit def dao_add_user_to_organization(organization_id, user_id): organization = dao_get_organization_by_id(organization_id) - user = User.query.filter_by(id=user_id).one() + stmt = select(User).filter_by(id=user_id) + user = db.session.execute(stmt).scalars().one() user.organizations.append(organization) db.session.add(organization) return user From 9174efe53597e6502f9d09b458f90515da541d8b Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 07:43:47 -0700 Subject: [PATCH 109/128] upgrade organization_dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 7fde65a2a..366508cf0 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -23,7 +23,8 @@ def dao_count_organizations_with_live_services(): Service.count_as_live.is_(True), ) ) - return db.session.execute(stmt).distinct().scalar() or 0 + # TODO Need distinct here? + return db.session.execute(stmt).scalar() or 0 def dao_get_organization_services(organization_id): From fe85970a86e7e4f896df14d8db5266fb4dab8315 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 07:55:43 -0700 Subject: [PATCH 110/128] upgrade organization_dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 366508cf0..6f045144d 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -15,7 +15,7 @@ def dao_get_organizations(): def dao_count_organizations_with_live_services(): stmt = ( - select(func.count(Organization.id)) + select(func.count(Organization.id).distinct()).select_from(Organization) .join(Organization.services) .filter( Service.active.is_(True), From ca365858bbefbea96c7670f3142ef61a99b2ec43 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 08:05:23 -0700 Subject: [PATCH 111/128] upgrade organization_dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 6f045144d..c686e35a1 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -15,7 +15,7 @@ def dao_get_organizations(): def dao_count_organizations_with_live_services(): stmt = ( - select(func.count(Organization.id).distinct()).select_from(Organization) + select(func.count().distinct()).select_from(Organization) .join(Organization.services) .filter( Service.active.is_(True), From eac1122dae08ed6b1092bcf8a9d0be1121ae4ef8 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 08:14:12 -0700 Subject: [PATCH 112/128] upgrade organization_dao to sqlalchemy 2.0 --- app/dao/organization_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index c686e35a1..548327631 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -15,7 +15,7 @@ def dao_get_organizations(): def dao_count_organizations_with_live_services(): stmt = ( - select(func.count().distinct()).select_from(Organization) + select(func.count(func.distinct(Organization.id))) .join(Organization.services) .filter( Service.active.is_(True), From 5a48c359c6ba603cfdd08b84472967932f91e657 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 08:34:29 -0700 Subject: [PATCH 113/128] fix template folder dao --- app/dao/organization_dao.py | 5 +++-- app/dao/template_folder_dao.py | 10 ++++++--- app/dao/templates_dao.py | 40 +++++++++++++++++++--------------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 548327631..34afd98e0 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -1,4 +1,4 @@ -from sqlalchemy import delete, select +from sqlalchemy import delete, select, update from sqlalchemy.sql.expression import func from app import db @@ -69,7 +69,8 @@ def dao_create_organization(organization): @autocommit def dao_update_organization(organization_id, **kwargs): domains = kwargs.pop("domains", None) - num_updated = Organization.query.filter_by(id=organization_id).update(kwargs) + stmt = update(Organization).where(id=organization_id).values(**kwargs) + num_updated = db.session.execute(stmt).rowcount if isinstance(domains, list): stmt = delete(Domain).filter_by(organization_id=organization_id) diff --git a/app/dao/template_folder_dao.py b/app/dao/template_folder_dao.py index ae1224179..269f407e0 100644 --- a/app/dao/template_folder_dao.py +++ b/app/dao/template_folder_dao.py @@ -1,16 +1,20 @@ +from sqlalchemy import select + from app import db from app.dao.dao_utils import autocommit from app.models import TemplateFolder def dao_get_template_folder_by_id_and_service_id(template_folder_id, service_id): - return TemplateFolder.query.filter( + stmt = select(TemplateFolder).filter( TemplateFolder.id == template_folder_id, TemplateFolder.service_id == service_id - ).one() + ) + return db.session.execute(stmt).scalars().one() def dao_get_valid_template_folders_by_id(folder_ids): - return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all() + stmt = select(TemplateFolder).filter(TemplateFolder.id.in_(folder_ids)) + return db.session.execute(stmt).scalars().all() @autocommit diff --git a/app/dao/templates_dao.py b/app/dao/templates_dao.py index 55d4363d6..7c5d7459e 100644 --- a/app/dao/templates_dao.py +++ b/app/dao/templates_dao.py @@ -1,6 +1,6 @@ import uuid -from sqlalchemy import asc, desc +from sqlalchemy import asc, desc, select from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class @@ -46,24 +46,29 @@ def dao_redact_template(template, user_id): def dao_get_template_by_id_and_service_id(template_id, service_id, version=None): if version is not None: - return TemplateHistory.query.filter_by( + stmt = select(TemplateHistory).filter_by( id=template_id, hidden=False, service_id=service_id, version=version - ).one() - return Template.query.filter_by( + ) + return db.session.execute(stmt).scalars().one() + stmt = select(Template).filter_by( id=template_id, hidden=False, service_id=service_id - ).one() + ) + return db.session.execute(stmt).scalars().one() def dao_get_template_by_id(template_id, version=None): if version is not None: - return TemplateHistory.query.filter_by(id=template_id, version=version).one() - return Template.query.filter_by(id=template_id).one() + stmt = select(TemplateHistory).filter_by(id=template_id, version=version) + return db.session.execute(stmt).scalars().one() + stmt = select(Template).filter_by(id=template_id) + return db.session.execute(stmt).scalars().one() def dao_get_all_templates_for_service(service_id, template_type=None): if template_type is not None: - return ( - Template.query.filter_by( + stmt = ( + select(Template) + .filter_by( service_id=service_id, template_type=template_type, hidden=False, @@ -73,26 +78,27 @@ def dao_get_all_templates_for_service(service_id, template_type=None): asc(Template.name), asc(Template.template_type), ) - .all() ) - - return ( - Template.query.filter_by(service_id=service_id, hidden=False, archived=False) + return db.session.execute(stmt).scalars().all() + stmt = ( + select(Template) + .filter_by(service_id=service_id, hidden=False, archived=False) .order_by( asc(Template.name), asc(Template.template_type), ) - .all() ) + return db.session.execute(stmt).scalars().all() def dao_get_template_versions(service_id, template_id): - return ( - TemplateHistory.query.filter_by( + stmt = ( + select(TemplateHistory) + .filter_by( service_id=service_id, id=template_id, hidden=False, ) .order_by(desc(TemplateHistory.version)) - .all() ) + return db.session.execute(stmt).scalars().all() From b4a45da4d413e4ca8c388f9cb3307016712d0ce6 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 08:50:04 -0700 Subject: [PATCH 114/128] fix template folder dao --- app/dao/organization_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 34afd98e0..8bb10f659 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -69,7 +69,7 @@ def dao_create_organization(organization): @autocommit def dao_update_organization(organization_id, **kwargs): domains = kwargs.pop("domains", None) - stmt = update(Organization).where(id=organization_id).values(**kwargs) + stmt = update(Organization).where(Organization.id==organization_id).values(**kwargs) num_updated = db.session.execute(stmt).rowcount if isinstance(domains, list): From c8a758aff63ea998d547c6e7e57c53405e7e77b1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 08:52:50 -0700 Subject: [PATCH 115/128] fix template folder dao --- app/dao/organization_dao.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index 8bb10f659..f699e33bc 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -69,7 +69,9 @@ def dao_create_organization(organization): @autocommit def dao_update_organization(organization_id, **kwargs): domains = kwargs.pop("domains", None) - stmt = update(Organization).where(Organization.id==organization_id).values(**kwargs) + stmt = ( + update(Organization).where(Organization.id == organization_id).values(**kwargs) + ) num_updated = db.session.execute(stmt).rowcount if isinstance(domains, list): From 68b9d8a484beb43e6486052df4a819dba8b48c08 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 09:21:06 -0700 Subject: [PATCH 116/128] fix template folder dao --- .ds.baseline | 4 +- tests/app/dao/test_organization_dao.py | 29 +++-- tests/app/dao/test_services_dao.py | 174 +++++++++++++++---------- 3 files changed, 126 insertions(+), 81 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 1c279e018..329763bf5 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 265, + "line_number": 266, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-09-27T16:42:53Z" + "generated_at": "2024-10-14T16:21:01Z" } diff --git a/tests/app/dao/test_organization_dao.py b/tests/app/dao/test_organization_dao.py index edffdd1d4..fb2e01d85 100644 --- a/tests/app/dao/test_organization_dao.py +++ b/tests/app/dao/test_organization_dao.py @@ -1,6 +1,7 @@ import uuid import pytest +from sqlalchemy import select from sqlalchemy.exc import IntegrityError, SQLAlchemyError from app import db @@ -57,7 +58,8 @@ def test_get_organization_by_id_gets_correct_organization(notify_db_session): def test_update_organization(notify_db_session): create_organization() - organization = Organization.query.one() + stmt = select(Organization) + organization = db.session.execute(stmt).scalars().one() user = create_user() email_branding = create_email_branding() @@ -78,7 +80,8 @@ def test_update_organization(notify_db_session): dao_update_organization(organization.id, **data) - organization = Organization.query.one() + stmt = select(Organization) + organization = db.session.execute(stmt).scalars().one() for attribute, value in data.items(): assert getattr(organization, attribute) == value @@ -102,7 +105,8 @@ def test_update_organization_domains_lowercases( ): create_organization() - organization = Organization.query.one() + stmt = select(Organization) + organization = db.session.execute(stmt).scalars().one() # Seed some domains dao_update_organization(organization.id, domains=["123", "456"]) @@ -121,7 +125,8 @@ def test_update_organization_domains_lowercases_integrity_error( ): create_organization() - organization = Organization.query.one() + stmt = select(Organization) + organization = db.session.execute(stmt).scalars().one() # Seed some domains dao_update_organization(organization.id, domains=["123", "456"]) @@ -175,11 +180,11 @@ def test_update_organization_updates_the_service_org_type_if_org_type_is_provide assert sample_organization.organization_type == OrganizationType.FEDERAL assert sample_service.organization_type == OrganizationType.FEDERAL + stmt = select(Service.get_history_model()).filter_by( + id=sample_service.id, version=2 + ) assert ( - Service.get_history_model() - .query.filter_by(id=sample_service.id, version=2) - .one() - .organization_type + db.session.execute(stmt).scalars().one().organization_type == OrganizationType.FEDERAL ) @@ -229,11 +234,11 @@ def test_add_service_to_organization(sample_service, sample_organization): assert sample_organization.services[0].id == sample_service.id assert sample_service.organization_type == sample_organization.organization_type + stmt = select(Service.get_history_model()).filter_by( + id=sample_service.id, version=2 + ) assert ( - Service.get_history_model() - .query.filter_by(id=sample_service.id, version=2) - .one() - .organization_type + db.session.execute(stmt).scalars().one().organization_type == sample_organization.organization_type ) assert sample_service.organization_id == sample_organization.id diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index e590eb5b4..487df6a29 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -6,6 +6,7 @@ from unittest.mock import Mock import pytest import sqlalchemy from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound @@ -91,7 +92,7 @@ from tests.app.db import ( def test_create_service(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -101,7 +102,7 @@ def test_create_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 + assert service_query_count() == 1 service_db = Service.query.one() assert service_db.name == "service_name" assert service_db.id == service.id @@ -120,7 +121,7 @@ def test_create_service_with_organization(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert Service.query.count() == 0 + assert service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -130,9 +131,9 @@ def test_create_service_with_organization(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 + assert service_query_count() == 1 service_db = Service.query.one() - organization = Organization.query.get(organization.id) + organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -151,7 +152,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert Service.query.count() == 0 + assert service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -161,9 +162,9 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 + assert service_query_count() == 1 service_db = Service.query.one() - organization = Organization.query.get(organization.id) + organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -183,7 +184,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): def test_cannot_create_two_services_with_same_name(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert service_query_count() == 0 service1 = Service( name="service_name", email_from="email_from1", @@ -209,7 +210,7 @@ def test_cannot_create_two_services_with_same_name(notify_db_session): def test_cannot_create_two_services_with_same_email_from(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert service_query_count() == 0 service1 = Service( name="service_name1", email_from="email_from", @@ -235,7 +236,7 @@ def test_cannot_create_two_services_with_same_email_from(notify_db_session): def test_cannot_create_service_with_no_user(notify_db_session): user = create_user() - assert Service.query.count() == 0 + assert service_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -258,7 +259,7 @@ def test_should_add_user_to_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert user in Service.query.first().users + assert user in service_query_first().users new_user = User( name="Test User", email_address="new_user@digital.fake.gov", @@ -267,7 +268,7 @@ def test_should_add_user_to_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in Service.query.first().users + assert new_user in service_query_first().users def test_dao_add_user_to_service_sets_folder_permissions(sample_user, sample_service): @@ -314,7 +315,8 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a other_service_folder = create_template_folder(other_service) folder_permissions = [str(other_service_folder.id)] - assert ServiceUser.query.count() == 2 + stmt = select(ServiceUser) + assert db.session.execute(stmt).scalar() == 2 with pytest.raises(IntegrityError) as e: dao_add_user_to_service( @@ -326,7 +328,8 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a 'insert or update on table "user_folder_permissions" violates foreign key constraint' in str(e.value) ) - assert ServiceUser.query.count() == 2 + stmt = select(ServiceUser) + assert db.session.execute(stmt).scalar() == 2 def test_should_remove_user_from_service(notify_db_session): @@ -347,9 +350,9 @@ def test_should_remove_user_from_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in Service.query.first().users + assert new_user in service_query_first().users dao_remove_user_from_service(service, new_user) - assert new_user not in Service.query.first().users + assert new_user not in service_query_first().users def test_should_remove_user_from_service_exception(notify_db_session): @@ -668,8 +671,8 @@ def test_removing_all_permission_returns_service_with_no_permissions(notify_db_s def test_create_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 + assert service_query_count() == 0 + assert service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -678,10 +681,10 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses created_by=user, ) dao_create_service(service, user) - assert Service.query.count() == 1 - assert Service.get_history_model().query.count() == 1 + assert service_query_count() == 1 + assert service_history_query_count() == 1 - service_from_db = Service.query.first() + service_from_db = service_query_first() service_history = Service.get_history_model().query.first() assert service_from_db.id == service_history.id @@ -692,10 +695,25 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses assert service_from_db.created_by.id == service_history.created_by_id +def service_query_count(): + stmt = select(func.count()).select_from(Service) + return db.session.execute(stmt).scalar() or 0 + + +def service_query_first(): + stmt = select(Service) + return db.session.execute(stmt).scalars().first() + + +def service_history_query_count(): + stmt = select(func.count()).select_from(Service.get_history_model()) + return db.session.execute(stmt).scalar() or 0 + + def test_update_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 + assert service_query_count() == 0 + assert service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -705,17 +723,17 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses ) dao_create_service(service, user) - assert Service.query.count() == 1 - assert Service.query.first().version == 1 - assert Service.get_history_model().query.count() == 1 + assert service_query_count() == 1 + assert service_query_first().version == 1 + assert service_history_query_count() == 1 service.name = "updated_service_name" dao_update_service(service) - assert Service.query.count() == 1 - assert Service.get_history_model().query.count() == 2 + assert service_query_count() == 1 + assert service_history_query_count() == 2 - service_from_db = Service.query.first() + service_from_db = service_query_first() assert service_from_db.version == 2 @@ -736,8 +754,8 @@ def test_update_service_permission_creates_a_history_record_with_current_data( notify_db_session, ): user = create_user() - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 + assert service_query_count() == 0 + assert service_history_query_count() == 0 service = Service( name="service_name", email_from="email_from", @@ -755,17 +773,17 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ], ) - assert Service.query.count() == 1 + assert service_query_count() == 1 service.permissions.append( ServicePermission(service_id=service.id, permission=ServicePermissionType.EMAIL) ) dao_update_service(service) - assert Service.query.count() == 1 - assert Service.get_history_model().query.count() == 2 + assert service_query_count() == 1 + assert service_history_query_count() == 2 - service_from_db = Service.query.first() + service_from_db = service_query_first() assert service_from_db.version == 2 @@ -784,10 +802,10 @@ def test_update_service_permission_creates_a_history_record_with_current_data( service.permissions.remove(permission) dao_update_service(service) - assert Service.query.count() == 1 - assert Service.get_history_model().query.count() == 3 + assert service_query_count() == 1 + assert service_history_query_count() == 3 - service_from_db = Service.query.first() + service_from_db = service_query_first() assert service_from_db.version == 3 _assert_service_permissions( service.permissions, @@ -810,8 +828,8 @@ def test_update_service_permission_creates_a_history_record_with_current_data( def test_create_service_and_history_is_transactional(notify_db_session): user = create_user() - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 + assert service_query_count() == 0 + assert service_history_query_count() == 0 service = Service( name=None, email_from="email_from", @@ -828,8 +846,8 @@ def test_create_service_and_history_is_transactional(notify_db_session): in str(seeei) ) - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 + assert service_query_count() == 0 + assert service_history_query_count() == 0 def test_delete_service_and_associated_objects(notify_db_session): @@ -846,7 +864,8 @@ def test_delete_service_and_associated_objects(notify_db_session): create_invited_user(service=service) user.organizations = [organization] - assert ServicePermission.query.count() == len( + stmt = select(ServicePermission) + assert db.session.execute(stmt).scalar() == len( ( ServicePermissionType.SMS, ServicePermissionType.EMAIL, @@ -855,21 +874,41 @@ def test_delete_service_and_associated_objects(notify_db_session): ) delete_service_and_all_associated_db_objects(service) - assert VerifyCode.query.count() == 0 - assert ApiKey.query.count() == 0 - assert ApiKey.get_history_model().query.count() == 0 - assert Template.query.count() == 0 - assert TemplateHistory.query.count() == 0 - assert Job.query.count() == 0 - assert Notification.query.count() == 0 - assert Permission.query.count() == 0 - assert User.query.count() == 0 - assert InvitedUser.query.count() == 0 - assert Service.query.count() == 0 - assert Service.get_history_model().query.count() == 0 - assert ServicePermission.query.count() == 0 + stmt = select(VerifyCode) + assert db.session.execute(stmt).scalar() is None + stmt = select(ApiKey) + assert db.session.execute(stmt).scalar() is None + stmt = select(ApiKey.get_history_model()) + assert db.session.execute(stmt).scalar() is None + + stmt = select(Template) + assert db.session.execute(stmt).scalar() is None + + stmt = select(TemplateHistory) + assert db.session.execute(stmt).scalar() is None + + stmt = select(Job) + assert db.session.execute(stmt).scalar() is None + + stmt = select(Notification) + assert db.session.execute(stmt).scalar() is None + + stmt = select(Permission) + assert db.session.execute(stmt).scalar() is None + + stmt = select(User) + assert db.session.execute(stmt).scalar() is None + + stmt = select(InvitedUser) + assert db.session.execute(stmt).scalar() is None + stmt = select(ServicePermission) + assert db.session.execute(stmt).scalar() is None + + assert service_query_count() == 0 + assert service_history_query_count() == 0 # the organization hasn't been deleted - assert Organization.query.count() == 1 + stmt = select(Organization) + assert db.session.execute(stmt).scalar() == 1 def test_add_existing_user_to_another_service_doesnot_change_old_permissions( @@ -956,9 +995,10 @@ def test_fetch_stats_filters_on_service(notify_db_session): def test_fetch_stats_ignores_historical_notification_data(sample_template): create_notification_history(template=sample_template) - - assert Notification.query.count() == 0 - assert NotificationHistory.query.count() == 1 + stmt = select(Notification) + assert db.session.execute(stmt).scalar() is None + stmt = select(NotificationHistory) + assert db.session.execute(stmt).scalar() == 1 stats = dao_fetch_todays_stats_for_service(sample_template.service_id) assert len(stats) == 0 @@ -1316,7 +1356,7 @@ def test_dao_fetch_todays_stats_for_all_services_can_exclude_from_test_key( def test_dao_suspend_service_with_no_api_keys(notify_db_session): service = create_service() dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = db.session.get(Service, service.id) assert not service.active assert service.name == service.name assert service.api_keys == [] @@ -1329,11 +1369,11 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys( service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = db.session.get(Service, service.id) assert not service.active assert service.name == service.name - api_key = ApiKey.query.get(api_key.id) + api_key = db.session.get(ApiKey, api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) @@ -1344,13 +1384,13 @@ def test_dao_resume_service_marks_service_as_active_and_api_keys_are_still_revok service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = Service.query.get(service.id) + service = db.session.get(Service, service.id) assert not service.active dao_resume_service(service.id) - assert Service.query.get(service.id).active + assert db.session.get(Service, service.id).active - api_key = ApiKey.query.get(api_key.id) + api_key = db.session.get(ApiKey, api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) From 1eb46b39b1b09f44ea781c68029d6d93bc581a73 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 09:32:01 -0700 Subject: [PATCH 117/128] fix template folder dao --- tests/app/dao/test_services_dao.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 487df6a29..8640df2a5 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -315,7 +315,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a other_service_folder = create_template_folder(other_service) folder_permissions = [str(other_service_folder.id)] - stmt = select(ServiceUser) + stmt = select(func.count()).select_from(ServiceUser) assert db.session.execute(stmt).scalar() == 2 with pytest.raises(IntegrityError) as e: @@ -328,7 +328,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a 'insert or update on table "user_folder_permissions" violates foreign key constraint' in str(e.value) ) - stmt = select(ServiceUser) + stmt = select(func.count()).select_from(ServiceUser) assert db.session.execute(stmt).scalar() == 2 @@ -864,7 +864,7 @@ def test_delete_service_and_associated_objects(notify_db_session): create_invited_user(service=service) user.organizations = [organization] - stmt = select(ServicePermission) + stmt = select(func.count()).select_from(ServicePermission) assert db.session.execute(stmt).scalar() == len( ( ServicePermissionType.SMS, @@ -907,7 +907,7 @@ def test_delete_service_and_associated_objects(notify_db_session): assert service_query_count() == 0 assert service_history_query_count() == 0 # the organization hasn't been deleted - stmt = select(Organization) + stmt = select(func.count()).select_from(Organization) assert db.session.execute(stmt).scalar() == 1 @@ -997,7 +997,7 @@ def test_fetch_stats_ignores_historical_notification_data(sample_template): create_notification_history(template=sample_template) stmt = select(Notification) assert db.session.execute(stmt).scalar() is None - stmt = select(NotificationHistory) + stmt = select(func.count()).select_from(NotificationHistory) assert db.session.execute(stmt).scalar() == 1 stats = dao_fetch_todays_stats_for_service(sample_template.service_id) From ef5a45d6a255eb1b47c718708515deecca302fbb Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 10:28:44 -0700 Subject: [PATCH 118/128] fix template folder dao --- .ds.baseline | 4 +- tests/app/dao/test_services_dao.py | 67 +++++++++++++----------------- 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 329763bf5..1a5a9727b 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 266, + "line_number": 269, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-14T16:21:01Z" + "generated_at": "2024-10-14T17:28:40Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 8640df2a5..c6d19c322 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -103,7 +103,8 @@ def test_create_service(notify_db_session): ) dao_create_service(service, user) assert service_query_count() == 1 - service_db = Service.query.one() + stmt = select(Service) + service_db = db.session.execute(stmt).scalars().one() assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -132,7 +133,8 @@ def test_create_service_with_organization(notify_db_session): ) dao_create_service(service, user) assert service_query_count() == 1 - service_db = Service.query.one() + stmt = select(Service) + service_db = db.session.execute(stmt).scalars().one() organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id @@ -163,7 +165,8 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): ) dao_create_service(service, user) assert service_query_count() == 1 - service_db = Service.query.one() + stmt = select(Service) + service_db = db.session.execute(stmt).scalars().one() organization = db.session.get(Organization, organization.id) assert service_db.name == "service_name" assert service_db.id == service.id @@ -385,11 +388,12 @@ def test_should_remove_user_from_service_exception(notify_db_session): def test_removing_a_user_from_a_service_deletes_their_permissions( sample_user, sample_service ): - assert len(Permission.query.all()) == 7 + stmt = select(Permission) + assert len(db.session.execute(stmt).scalars().all()) == 7 dao_remove_user_from_service(sample_service, sample_user) - assert Permission.query.all() == [] + assert db.session.execute(stmt).scalars().all() == [] def test_removing_a_user_from_a_service_deletes_their_folder_permissions_for_that_service( @@ -685,7 +689,9 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses assert service_history_query_count() == 1 service_from_db = service_query_first() - service_history = Service.get_history_model().query.first() + stmt = select(Service.get_history_model()) + + service_history = db.session.execute(stmt).scalars().first() assert service_from_db.id == service_history.id assert service_from_db.name == service_history.name @@ -737,17 +743,10 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses assert service_from_db.version == 2 - assert ( - Service.get_history_model().query.filter_by(name="service_name").one().version - == 1 - ) - assert ( - Service.get_history_model() - .query.filter_by(name="updated_service_name") - .one() - .version - == 2 - ) + stmt = select(Service.get_history_model()).filter_by(name="service_name") + assert db.session.execute(stmt).scalars().one().version == 1 + stmt = select(Service.get_history_model()).filter_by(name="updated_service_name") + assert db.session.execute(stmt).scalars().one().version == 2 def test_update_service_permission_creates_a_history_record_with_current_data( @@ -815,12 +814,12 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ), ) - history = ( - Service.get_history_model() - .query.filter_by(name="service_name") + stmt = ( + select(Service.get_history_model()) + .filter_by(name="service_name") .order_by("version") - .all() ) + history = db.session.execute(stmt).scalars().all() assert len(history) == 3 assert history[2].version == 3 @@ -926,9 +925,8 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_one, user) assert user.id == service_one.users[0].id - test_user_permissions = Permission.query.filter_by( - service=service_one, user=user - ).all() + stmt = select(Permission).filter_by(service=service_one, user=user) + test_user_permissions = db.session.execute(stmt).scalars().all() assert len(test_user_permissions) == 7 other_user = User( @@ -948,14 +946,11 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_two, other_user) assert other_user.id == service_two.users[0].id - other_user_permissions = Permission.query.filter_by( - service=service_two, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_two, user=other_user) + other_user_permissions = db.session.execute(stmt).scalars().all() assert len(other_user_permissions) == 7 - - other_user_service_one_permissions = Permission.query.filter_by( - service=service_one, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_one, user=other_user) + other_user_service_one_permissions = db.session.execute(stmt).scalars().all() assert len(other_user_service_one_permissions) == 0 # adding the other_user to service_one should leave all other_user permissions on service_two intact @@ -965,14 +960,12 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_add_user_to_service(service_one, other_user, permissions=permissions) - other_user_service_one_permissions = Permission.query.filter_by( - service=service_one, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_one, user=other_user) + other_user_service_one_permissions = db.session.execute(stmt).scalars().all() assert len(other_user_service_one_permissions) == 2 - other_user_service_two_permissions = Permission.query.filter_by( - service=service_two, user=other_user - ).all() + stmt = select(Permission).filter_by(service=service_two, user=other_user) + other_user_service_two_permissions = db.session.execute(stmt).scalars().all() assert len(other_user_service_two_permissions) == 7 From 1c42508be72cdf0781939dcebb2c0a07b3098679 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 10:46:51 -0700 Subject: [PATCH 119/128] fix template folder dao --- .ds.baseline | 4 +- tests/app/dao/test_services_dao.py | 241 ++++++++++------------ tests/app/dao/test_template_folder_dao.py | 6 +- tests/app/dao/test_templates_dao.py | 67 +++--- 4 files changed, 154 insertions(+), 164 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 1a5a9727b..d8f938338 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -239,7 +239,7 @@ "filename": "tests/app/dao/test_services_dao.py", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 269, + "line_number": 265, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2024-10-14T17:28:40Z" + "generated_at": "2024-10-14T17:46:47Z" } diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index c6d19c322..e590eb5b4 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -6,7 +6,6 @@ from unittest.mock import Mock import pytest import sqlalchemy from freezegun import freeze_time -from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound @@ -92,7 +91,7 @@ from tests.app.db import ( def test_create_service(notify_db_session): user = create_user() - assert service_query_count() == 0 + assert Service.query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -102,9 +101,8 @@ def test_create_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert service_query_count() == 1 - stmt = select(Service) - service_db = db.session.execute(stmt).scalars().one() + assert Service.query.count() == 1 + service_db = Service.query.one() assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -122,7 +120,7 @@ def test_create_service_with_organization(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert service_query_count() == 0 + assert Service.query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -132,10 +130,9 @@ def test_create_service_with_organization(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert service_query_count() == 1 - stmt = select(Service) - service_db = db.session.execute(stmt).scalars().one() - organization = db.session.get(Organization, organization.id) + assert Service.query.count() == 1 + service_db = Service.query.one() + organization = Organization.query.get(organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -154,7 +151,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): organization_type=OrganizationType.STATE, domains=["local-authority.gov.uk"], ) - assert service_query_count() == 0 + assert Service.query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -164,10 +161,9 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert service_query_count() == 1 - stmt = select(Service) - service_db = db.session.execute(stmt).scalars().one() - organization = db.session.get(Organization, organization.id) + assert Service.query.count() == 1 + service_db = Service.query.one() + organization = Organization.query.get(organization.id) assert service_db.name == "service_name" assert service_db.id == service.id assert service_db.email_from == "email_from" @@ -187,7 +183,7 @@ def test_fetch_service_by_id_with_api_keys(notify_db_session): def test_cannot_create_two_services_with_same_name(notify_db_session): user = create_user() - assert service_query_count() == 0 + assert Service.query.count() == 0 service1 = Service( name="service_name", email_from="email_from1", @@ -213,7 +209,7 @@ def test_cannot_create_two_services_with_same_name(notify_db_session): def test_cannot_create_two_services_with_same_email_from(notify_db_session): user = create_user() - assert service_query_count() == 0 + assert Service.query.count() == 0 service1 = Service( name="service_name1", email_from="email_from", @@ -239,7 +235,7 @@ def test_cannot_create_two_services_with_same_email_from(notify_db_session): def test_cannot_create_service_with_no_user(notify_db_session): user = create_user() - assert service_query_count() == 0 + assert Service.query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -262,7 +258,7 @@ def test_should_add_user_to_service(notify_db_session): created_by=user, ) dao_create_service(service, user) - assert user in service_query_first().users + assert user in Service.query.first().users new_user = User( name="Test User", email_address="new_user@digital.fake.gov", @@ -271,7 +267,7 @@ def test_should_add_user_to_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in service_query_first().users + assert new_user in Service.query.first().users def test_dao_add_user_to_service_sets_folder_permissions(sample_user, sample_service): @@ -318,8 +314,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a other_service_folder = create_template_folder(other_service) folder_permissions = [str(other_service_folder.id)] - stmt = select(func.count()).select_from(ServiceUser) - assert db.session.execute(stmt).scalar() == 2 + assert ServiceUser.query.count() == 2 with pytest.raises(IntegrityError) as e: dao_add_user_to_service( @@ -331,8 +326,7 @@ def test_dao_add_user_to_service_raises_error_if_adding_folder_permissions_for_a 'insert or update on table "user_folder_permissions" violates foreign key constraint' in str(e.value) ) - stmt = select(func.count()).select_from(ServiceUser) - assert db.session.execute(stmt).scalar() == 2 + assert ServiceUser.query.count() == 2 def test_should_remove_user_from_service(notify_db_session): @@ -353,9 +347,9 @@ def test_should_remove_user_from_service(notify_db_session): ) save_model_user(new_user, validated_email_access=True) dao_add_user_to_service(service, new_user) - assert new_user in service_query_first().users + assert new_user in Service.query.first().users dao_remove_user_from_service(service, new_user) - assert new_user not in service_query_first().users + assert new_user not in Service.query.first().users def test_should_remove_user_from_service_exception(notify_db_session): @@ -388,12 +382,11 @@ def test_should_remove_user_from_service_exception(notify_db_session): def test_removing_a_user_from_a_service_deletes_their_permissions( sample_user, sample_service ): - stmt = select(Permission) - assert len(db.session.execute(stmt).scalars().all()) == 7 + assert len(Permission.query.all()) == 7 dao_remove_user_from_service(sample_service, sample_user) - assert db.session.execute(stmt).scalars().all() == [] + assert Permission.query.all() == [] def test_removing_a_user_from_a_service_deletes_their_folder_permissions_for_that_service( @@ -675,8 +668,8 @@ def test_removing_all_permission_returns_service_with_no_permissions(notify_db_s def test_create_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -685,13 +678,11 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses created_by=user, ) dao_create_service(service, user) - assert service_query_count() == 1 - assert service_history_query_count() == 1 + assert Service.query.count() == 1 + assert Service.get_history_model().query.count() == 1 - service_from_db = service_query_first() - stmt = select(Service.get_history_model()) - - service_history = db.session.execute(stmt).scalars().first() + service_from_db = Service.query.first() + service_history = Service.get_history_model().query.first() assert service_from_db.id == service_history.id assert service_from_db.name == service_history.name @@ -701,25 +692,10 @@ def test_create_service_creates_a_history_record_with_current_data(notify_db_ses assert service_from_db.created_by.id == service_history.created_by_id -def service_query_count(): - stmt = select(func.count()).select_from(Service) - return db.session.execute(stmt).scalar() or 0 - - -def service_query_first(): - stmt = select(Service) - return db.session.execute(stmt).scalars().first() - - -def service_history_query_count(): - stmt = select(func.count()).select_from(Service.get_history_model()) - return db.session.execute(stmt).scalar() or 0 - - def test_update_service_creates_a_history_record_with_current_data(notify_db_session): user = create_user() - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -729,32 +705,39 @@ def test_update_service_creates_a_history_record_with_current_data(notify_db_ses ) dao_create_service(service, user) - assert service_query_count() == 1 - assert service_query_first().version == 1 - assert service_history_query_count() == 1 + assert Service.query.count() == 1 + assert Service.query.first().version == 1 + assert Service.get_history_model().query.count() == 1 service.name = "updated_service_name" dao_update_service(service) - assert service_query_count() == 1 - assert service_history_query_count() == 2 + assert Service.query.count() == 1 + assert Service.get_history_model().query.count() == 2 - service_from_db = service_query_first() + service_from_db = Service.query.first() assert service_from_db.version == 2 - stmt = select(Service.get_history_model()).filter_by(name="service_name") - assert db.session.execute(stmt).scalars().one().version == 1 - stmt = select(Service.get_history_model()).filter_by(name="updated_service_name") - assert db.session.execute(stmt).scalars().one().version == 2 + assert ( + Service.get_history_model().query.filter_by(name="service_name").one().version + == 1 + ) + assert ( + Service.get_history_model() + .query.filter_by(name="updated_service_name") + .one() + .version + == 2 + ) def test_update_service_permission_creates_a_history_record_with_current_data( notify_db_session, ): user = create_user() - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 service = Service( name="service_name", email_from="email_from", @@ -772,17 +755,17 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ], ) - assert service_query_count() == 1 + assert Service.query.count() == 1 service.permissions.append( ServicePermission(service_id=service.id, permission=ServicePermissionType.EMAIL) ) dao_update_service(service) - assert service_query_count() == 1 - assert service_history_query_count() == 2 + assert Service.query.count() == 1 + assert Service.get_history_model().query.count() == 2 - service_from_db = service_query_first() + service_from_db = Service.query.first() assert service_from_db.version == 2 @@ -801,10 +784,10 @@ def test_update_service_permission_creates_a_history_record_with_current_data( service.permissions.remove(permission) dao_update_service(service) - assert service_query_count() == 1 - assert service_history_query_count() == 3 + assert Service.query.count() == 1 + assert Service.get_history_model().query.count() == 3 - service_from_db = service_query_first() + service_from_db = Service.query.first() assert service_from_db.version == 3 _assert_service_permissions( service.permissions, @@ -814,12 +797,12 @@ def test_update_service_permission_creates_a_history_record_with_current_data( ), ) - stmt = ( - select(Service.get_history_model()) - .filter_by(name="service_name") + history = ( + Service.get_history_model() + .query.filter_by(name="service_name") .order_by("version") + .all() ) - history = db.session.execute(stmt).scalars().all() assert len(history) == 3 assert history[2].version == 3 @@ -827,8 +810,8 @@ def test_update_service_permission_creates_a_history_record_with_current_data( def test_create_service_and_history_is_transactional(notify_db_session): user = create_user() - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 service = Service( name=None, email_from="email_from", @@ -845,8 +828,8 @@ def test_create_service_and_history_is_transactional(notify_db_session): in str(seeei) ) - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 def test_delete_service_and_associated_objects(notify_db_session): @@ -863,8 +846,7 @@ def test_delete_service_and_associated_objects(notify_db_session): create_invited_user(service=service) user.organizations = [organization] - stmt = select(func.count()).select_from(ServicePermission) - assert db.session.execute(stmt).scalar() == len( + assert ServicePermission.query.count() == len( ( ServicePermissionType.SMS, ServicePermissionType.EMAIL, @@ -873,41 +855,21 @@ def test_delete_service_and_associated_objects(notify_db_session): ) delete_service_and_all_associated_db_objects(service) - stmt = select(VerifyCode) - assert db.session.execute(stmt).scalar() is None - stmt = select(ApiKey) - assert db.session.execute(stmt).scalar() is None - stmt = select(ApiKey.get_history_model()) - assert db.session.execute(stmt).scalar() is None - - stmt = select(Template) - assert db.session.execute(stmt).scalar() is None - - stmt = select(TemplateHistory) - assert db.session.execute(stmt).scalar() is None - - stmt = select(Job) - assert db.session.execute(stmt).scalar() is None - - stmt = select(Notification) - assert db.session.execute(stmt).scalar() is None - - stmt = select(Permission) - assert db.session.execute(stmt).scalar() is None - - stmt = select(User) - assert db.session.execute(stmt).scalar() is None - - stmt = select(InvitedUser) - assert db.session.execute(stmt).scalar() is None - stmt = select(ServicePermission) - assert db.session.execute(stmt).scalar() is None - - assert service_query_count() == 0 - assert service_history_query_count() == 0 + assert VerifyCode.query.count() == 0 + assert ApiKey.query.count() == 0 + assert ApiKey.get_history_model().query.count() == 0 + assert Template.query.count() == 0 + assert TemplateHistory.query.count() == 0 + assert Job.query.count() == 0 + assert Notification.query.count() == 0 + assert Permission.query.count() == 0 + assert User.query.count() == 0 + assert InvitedUser.query.count() == 0 + assert Service.query.count() == 0 + assert Service.get_history_model().query.count() == 0 + assert ServicePermission.query.count() == 0 # the organization hasn't been deleted - stmt = select(func.count()).select_from(Organization) - assert db.session.execute(stmt).scalar() == 1 + assert Organization.query.count() == 1 def test_add_existing_user_to_another_service_doesnot_change_old_permissions( @@ -925,8 +887,9 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_one, user) assert user.id == service_one.users[0].id - stmt = select(Permission).filter_by(service=service_one, user=user) - test_user_permissions = db.session.execute(stmt).scalars().all() + test_user_permissions = Permission.query.filter_by( + service=service_one, user=user + ).all() assert len(test_user_permissions) == 7 other_user = User( @@ -946,11 +909,14 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_create_service(service_two, other_user) assert other_user.id == service_two.users[0].id - stmt = select(Permission).filter_by(service=service_two, user=other_user) - other_user_permissions = db.session.execute(stmt).scalars().all() + other_user_permissions = Permission.query.filter_by( + service=service_two, user=other_user + ).all() assert len(other_user_permissions) == 7 - stmt = select(Permission).filter_by(service=service_one, user=other_user) - other_user_service_one_permissions = db.session.execute(stmt).scalars().all() + + other_user_service_one_permissions = Permission.query.filter_by( + service=service_one, user=other_user + ).all() assert len(other_user_service_one_permissions) == 0 # adding the other_user to service_one should leave all other_user permissions on service_two intact @@ -960,12 +926,14 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions( dao_add_user_to_service(service_one, other_user, permissions=permissions) - stmt = select(Permission).filter_by(service=service_one, user=other_user) - other_user_service_one_permissions = db.session.execute(stmt).scalars().all() + other_user_service_one_permissions = Permission.query.filter_by( + service=service_one, user=other_user + ).all() assert len(other_user_service_one_permissions) == 2 - stmt = select(Permission).filter_by(service=service_two, user=other_user) - other_user_service_two_permissions = db.session.execute(stmt).scalars().all() + other_user_service_two_permissions = Permission.query.filter_by( + service=service_two, user=other_user + ).all() assert len(other_user_service_two_permissions) == 7 @@ -988,10 +956,9 @@ def test_fetch_stats_filters_on_service(notify_db_session): def test_fetch_stats_ignores_historical_notification_data(sample_template): create_notification_history(template=sample_template) - stmt = select(Notification) - assert db.session.execute(stmt).scalar() is None - stmt = select(func.count()).select_from(NotificationHistory) - assert db.session.execute(stmt).scalar() == 1 + + assert Notification.query.count() == 0 + assert NotificationHistory.query.count() == 1 stats = dao_fetch_todays_stats_for_service(sample_template.service_id) assert len(stats) == 0 @@ -1349,7 +1316,7 @@ def test_dao_fetch_todays_stats_for_all_services_can_exclude_from_test_key( def test_dao_suspend_service_with_no_api_keys(notify_db_session): service = create_service() dao_suspend_service(service.id) - service = db.session.get(Service, service.id) + service = Service.query.get(service.id) assert not service.active assert service.name == service.name assert service.api_keys == [] @@ -1362,11 +1329,11 @@ def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys( service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = db.session.get(Service, service.id) + service = Service.query.get(service.id) assert not service.active assert service.name == service.name - api_key = db.session.get(ApiKey, api_key.id) + api_key = ApiKey.query.get(api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) @@ -1377,13 +1344,13 @@ def test_dao_resume_service_marks_service_as_active_and_api_keys_are_still_revok service = create_service() api_key = create_api_key(service=service) dao_suspend_service(service.id) - service = db.session.get(Service, service.id) + service = Service.query.get(service.id) assert not service.active dao_resume_service(service.id) - assert db.session.get(Service, service.id).active + assert Service.query.get(service.id).active - api_key = db.session.get(ApiKey, api_key.id) + api_key = ApiKey.query.get(api_key.id) assert api_key.expiry_date == datetime(2001, 1, 1, 23, 59, 00) diff --git a/tests/app/dao/test_template_folder_dao.py b/tests/app/dao/test_template_folder_dao.py index 17b03e5df..2a872e775 100644 --- a/tests/app/dao/test_template_folder_dao.py +++ b/tests/app/dao/test_template_folder_dao.py @@ -1,3 +1,5 @@ +from sqlalchemy import select + from app import db from app.dao.service_user_dao import dao_get_service_user from app.dao.template_folder_dao import ( @@ -17,5 +19,5 @@ def test_dao_delete_template_folder_deletes_user_folder_permissions( dao_update_template_folder(folder) dao_delete_template_folder(folder) - - assert db.session.query(user_folder_permissions).all() == [] + stmt = select(user_folder_permissions) + assert db.session.execute(stmt).scalars().all() == [] diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index bfe0e59d1..8da454d30 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -2,8 +2,10 @@ from datetime import datetime import pytest from freezegun import freeze_time +from sqlalchemy import func, select from sqlalchemy.orm.exc import NoResultFound +from app import db from app.dao.templates_dao import ( dao_create_template, dao_get_all_templates_for_service, @@ -17,6 +19,16 @@ from app.models import Template, TemplateHistory, TemplateRedacted from tests.app.db import create_template +def template_query_count(): + stmt = select(func.count()).select_from(Template) + return db.session.execute(stmt).scalar or 0 + + +def template_history_query_count(): + stmt = select(func.count()).select_from(TemplateHistory) + return db.session.execute(stmt).scalar or 0 + + @pytest.mark.parametrize( "template_type, subject", [ @@ -37,7 +49,7 @@ def test_create_template(sample_service, sample_user, template_type, subject): template = Template(**data) dao_create_template(template) - assert Template.query.count() == 1 + assert template_query_count() == 1 assert len(dao_get_all_templates_for_service(sample_service.id)) == 1 assert ( dao_get_all_templates_for_service(sample_service.id)[0].name @@ -50,11 +62,13 @@ def test_create_template(sample_service, sample_user, template_type, subject): def test_create_template_creates_redact_entry(sample_service): - assert TemplateRedacted.query.count() == 0 + stmt = select(func.count()).select_from(TemplateRedacted) + assert db.session.execute(stmt).scalar() is None template = create_template(sample_service) - redacted = TemplateRedacted.query.one() + stmt = select(TemplateRedacted) + redacted = db.session.execute(stmt).scalars().one() assert redacted.template_id == template.id assert redacted.redact_personalisation is False assert redacted.updated_by_id == sample_service.created_by_id @@ -79,7 +93,8 @@ def test_update_template(sample_service, sample_user): def test_redact_template(sample_template): - redacted = TemplateRedacted.query.one() + stmt = select(TemplateRedacted) + redacted = db.session.execute(stmt).scalars().one() assert redacted.template_id == sample_template.id assert redacted.redact_personalisation is False @@ -96,7 +111,7 @@ def test_get_all_templates_for_service(service_factory): service_1 = service_factory.get("service 1", email_from="service.1") service_2 = service_factory.get("service 2", email_from="service.2") - assert Template.query.count() == 2 + assert template_query_count() == 2 assert len(dao_get_all_templates_for_service(service_1.id)) == 1 assert len(dao_get_all_templates_for_service(service_2.id)) == 1 @@ -119,7 +134,7 @@ def test_get_all_templates_for_service(service_factory): content="Template content", ) - assert Template.query.count() == 5 + assert template_query_count() == 5 assert len(dao_get_all_templates_for_service(service_1.id)) == 3 assert len(dao_get_all_templates_for_service(service_2.id)) == 2 @@ -144,7 +159,7 @@ def test_get_all_templates_for_service_is_alphabetised(sample_service): service=sample_service, ) - assert Template.query.count() == 3 + assert template_query_count() == 3 assert ( dao_get_all_templates_for_service(sample_service.id)[0].name == "Sample Template 1" @@ -171,7 +186,7 @@ def test_get_all_templates_for_service_is_alphabetised(sample_service): def test_get_all_returns_empty_list_if_no_templates(sample_service): - assert Template.query.count() == 0 + assert template_query_count() == 0 assert len(dao_get_all_templates_for_service(sample_service.id)) == 0 @@ -257,8 +272,8 @@ def test_get_template_by_id_and_service_returns_none_if_no_template( def test_create_template_creates_a_history_record_with_current_data( sample_service, sample_user ): - assert Template.query.count() == 0 - assert TemplateHistory.query.count() == 0 + assert template_query_count() == 0 + assert template_history_query_count() == 0 data = { "name": "Sample Template", "template_type": TemplateType.EMAIL, @@ -270,10 +285,12 @@ def test_create_template_creates_a_history_record_with_current_data( template = Template(**data) dao_create_template(template) - assert Template.query.count() == 1 + assert template_query_count() == 1 - template_from_db = Template.query.first() - template_history = TemplateHistory.query.first() + stmt = select(Template) + template_from_db = db.session.execute(stmt).scalars().first() + stmt = select(TemplateHistory) + template_history = db.session.execute(stmt).scalars().first() assert template_from_db.id == template_history.id assert template_from_db.name == template_history.name @@ -286,8 +303,8 @@ def test_create_template_creates_a_history_record_with_current_data( def test_update_template_creates_a_history_record_with_current_data( sample_service, sample_user ): - assert Template.query.count() == 0 - assert TemplateHistory.query.count() == 0 + assert template_query_count() == 0 + assert template_history_query_count() == 0 data = { "name": "Sample Template", "template_type": TemplateType.EMAIL, @@ -301,22 +318,26 @@ def test_update_template_creates_a_history_record_with_current_data( created = dao_get_all_templates_for_service(sample_service.id)[0] assert created.name == "Sample Template" - assert Template.query.count() == 1 - assert Template.query.first().version == 1 - assert TemplateHistory.query.count() == 1 + assert template_query_count() == 1 + stmt = select(Template) + assert db.session.execute(stmt).scalars().first().version == 1 + assert template_history_query_count() == 1 created.name = "new name" dao_update_template(created) - assert Template.query.count() == 1 - assert TemplateHistory.query.count() == 2 + assert template_query_count() == 1 + assert template_history_query_count() == 2 - template_from_db = Template.query.first() + stmt = select(Template) + template_from_db = db.session.execute(stmt).scalars().first() assert template_from_db.version == 2 - assert TemplateHistory.query.filter_by(name="Sample Template").one().version == 1 - assert TemplateHistory.query.filter_by(name="new name").one().version == 2 + stmt = select(TemplateHistory).filter_by(name="Sample Template") + assert db.session.execute(stmt).scalars().one().version == 1 + stmt = select(TemplateHistory).filter_by(name="new name") + assert db.session.execute(stmt).scalars().one().version == 2 def test_get_template_history_version(sample_user, sample_service, sample_template): From 758e12c901e77e9f0884824ff031e040b0273969 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 10:55:10 -0700 Subject: [PATCH 120/128] fix template folder dao --- tests/app/dao/test_templates_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index 8da454d30..5fe603a64 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -21,12 +21,12 @@ from tests.app.db import create_template def template_query_count(): stmt = select(func.count()).select_from(Template) - return db.session.execute(stmt).scalar or 0 + return db.session.execute(stmt).scalar() or 0 def template_history_query_count(): stmt = select(func.count()).select_from(TemplateHistory) - return db.session.execute(stmt).scalar or 0 + return db.session.execute(stmt).scalar() or 0 @pytest.mark.parametrize( From 4a7e4c79a83d5fdcea353cb5ff782666f43c0c56 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 14 Oct 2024 11:03:17 -0700 Subject: [PATCH 121/128] fix template folder dao --- tests/app/dao/test_templates_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/dao/test_templates_dao.py b/tests/app/dao/test_templates_dao.py index 5fe603a64..734a29c0a 100644 --- a/tests/app/dao/test_templates_dao.py +++ b/tests/app/dao/test_templates_dao.py @@ -63,7 +63,7 @@ def test_create_template(sample_service, sample_user, template_type, subject): def test_create_template_creates_redact_entry(sample_service): stmt = select(func.count()).select_from(TemplateRedacted) - assert db.session.execute(stmt).scalar() is None + assert db.session.execute(stmt).scalar() == 0 template = create_template(sample_service) From 261ea6fe8f6fc17eec227b557c90264cbffeba72 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 21 Oct 2024 10:32:28 -0700 Subject: [PATCH 122/128] fix commented out code --- migrations/versions/0044_jobs_to_notification_hist.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/migrations/versions/0044_jobs_to_notification_hist.py b/migrations/versions/0044_jobs_to_notification_hist.py index e813833b4..3312d9a49 100644 --- a/migrations/versions/0044_jobs_to_notification_hist.py +++ b/migrations/versions/0044_jobs_to_notification_hist.py @@ -31,10 +31,10 @@ def upgrade(): # # go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d') # notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S') - # jobs = session.query(Job).join(Template).filter(Job.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091', + # stmt = select(Job).join(Template).filter(Job.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091', # Job.created_at >= go_live, # Job.created_at < notifications_history_start_date).all() - # + # jobs = db.session.execute(stmt).scalars().all() # for job in jobs: # for i in range(0, job.notifications_delivered): # notification = NotificationHistory(id=uuid.uuid4(), @@ -76,12 +76,11 @@ def downgrade(): # # go_live = datetime.datetime.strptime('2016-05-18', '%Y-%m-%d') # notifications_history_start_date = datetime.datetime.strptime('2016-06-26 23:21:55', '%Y-%m-%d %H:%M:%S') - # - # session.query(NotificationHistory).filter( + # stmt = delete(NotificationHistory).where( # NotificationHistory.created_at >= go_live, # NotificationHistory.service_id == '95316ff0-e555-462d-a6e7-95d26fbfd091', - # NotificationHistory.created_at < notifications_history_start_date).delete() - # + # NotificationHistory.created_at < notifications_history_start_date) + # session.execute(stmt) # session.commit() # ### end Alembic commands ### pass From 4c891de47c52bd46933880891c63a7922260b2f8 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 11 Oct 2024 13:46:53 -0400 Subject: [PATCH 123/128] Nonce stuff added. Signed-off-by: Cliff Hill --- app/service_invite/rest.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/service_invite/rest.py b/app/service_invite/rest.py index 5728b3ed5..e7d0d4b20 100644 --- a/app/service_invite/rest.py +++ b/app/service_invite/rest.py @@ -32,7 +32,7 @@ service_invite = Blueprint("service_invite", __name__) register_errors(service_invite) -def _create_service_invite(invited_user, invite_link_host): +def _create_service_invite(invited_user, nonce): template_id = current_app.config["INVITATION_EMAIL_TEMPLATE_ID"] @@ -40,12 +40,6 @@ def _create_service_invite(invited_user, invite_link_host): service = Service.query.get(current_app.config["NOTIFY_SERVICE_ID"]) - token = generate_token( - str(invited_user.email_address), - current_app.config["SECRET_KEY"], - current_app.config["DANGEROUS_SALT"], - ) - # The raw permissions are in the form "a,b,c,d" # but need to be in the form ["a", "b", "c", "d"] data = {} @@ -59,7 +53,8 @@ def _create_service_invite(invited_user, invite_link_host): data["invited_user_email"] = invited_user.email_address url = os.environ["LOGIN_DOT_GOV_REGISTRATION_URL"] - url = url.replace("NONCE", token) + + url = url.replace("NONCE", nonce) # handed from data sent from admin. user_data_url_safe = get_user_data_url_safe(data) @@ -94,10 +89,16 @@ def _create_service_invite(invited_user, invite_link_host): @service_invite.route("/service//invite", methods=["POST"]) def create_invited_user(service_id): request_json = request.get_json() + try: + nonce = request_json.pop("nonce") + except KeyError: + current_app.logger.exception("nonce not found in submitted data.") + raise + invited_user = invited_user_schema.load(request_json) save_invited_user(invited_user) - _create_service_invite(invited_user, request_json.get("invite_link_host")) + _create_service_invite(invited_user, nonce) return jsonify(data=invited_user_schema.dump(invited_user)), 201 From 0b648c98ddf833df722ba32cfd5e3749694ca7c8 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Mon, 21 Oct 2024 16:37:31 -0400 Subject: [PATCH 124/128] Fixed tests Signed-off-by: Cliff Hill --- tests/app/service_invite/test_service_invite_rest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/app/service_invite/test_service_invite_rest.py b/tests/app/service_invite/test_service_invite_rest.py index 07d0b4c23..5cea786f5 100644 --- a/tests/app/service_invite/test_service_invite_rest.py +++ b/tests/app/service_invite/test_service_invite_rest.py @@ -45,6 +45,7 @@ def test_create_invited_user( permissions="send_messages,manage_service,manage_api_keys", auth_type=AuthType.EMAIL, folder_permissions=["folder_1", "folder_2", "folder_3"], + nonce="FakeNonce", **extra_args, ) @@ -108,6 +109,7 @@ def test_create_invited_user_without_auth_type( "from_user": str(invite_from.id), "permissions": "send_messages,manage_service,manage_api_keys", "folder_permissions": [], + "nonce": "FakeNonce", } json_resp = admin_request.post( @@ -131,6 +133,7 @@ def test_create_invited_user_invalid_email(client, sample_service, mocker, fake_ "from_user": str(invite_from.id), "permissions": "send_messages,manage_service,manage_api_keys", "folder_permissions": [fake_uuid, fake_uuid], + "nonce": "FakeNonce", } data = json.dumps(data) From d5cc8b239f7d6b5e46f4067a2edfc2d6b3956ec6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Oct 2024 00:08:57 +0000 Subject: [PATCH 125/128] Bump werkzeug from 3.0.3 to 3.0.6 Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.0.6. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.0.6) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 60ce4d0ae..dcdb5290b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4519,13 +4519,13 @@ test = ["websockets"] [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.6" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.6-py3-none-any.whl", hash = "sha256:1bc0c2310d2fbb07b1dd1105eba2f7af72f322e1e455f2f93c993bee8c8a5f17"}, + {file = "werkzeug-3.0.6.tar.gz", hash = "sha256:a8dd59d4de28ca70471a34cba79bed5f7ef2e036a76b3ab0835474246eb41f8d"}, ] [package.dependencies] @@ -4803,4 +4803,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "42172a923e16c5b0965ab06f717d41e8491ee35f7be674091b38014c48b7a89e" +content-hash = "cf18ae74630e47eec18cc6c5fea9e554476809d20589d82c54a8d761bb2c3de0" diff --git a/pyproject.toml b/pyproject.toml index 3e3a78aed..99858c09e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ psycopg2-binary = "==2.9.9" pyjwt = "==2.8.0" python-dotenv = "==1.0.1" sqlalchemy = "==2.0.31" -werkzeug = "^3.0.3" +werkzeug = "^3.0.6" faker = "^26.0.0" async-timeout = "^4.0.3" bleach = "^6.1.0" From a9385107b033541a727c837567dbff0cea53de93 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 28 Oct 2024 12:50:11 -0700 Subject: [PATCH 126/128] code review feedback --- app/dao/notifications_dao.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 8659fca9b..1d07473c1 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -629,7 +629,6 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ Notification.status == NotificationStatus.CREATED, ) notifications = db.session.execute(stmt).scalars().all() - print(f"WE RETURN THIS FOR NOTIFICATIONS {notifications}") return notifications From 829d9020d8716e0874068bfe66bb3b42a251efd9 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 28 Oct 2024 13:03:50 -0700 Subject: [PATCH 127/128] code review feedback --- app/dao/organization_dao.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/dao/organization_dao.py b/app/dao/organization_dao.py index f699e33bc..668ac6c25 100644 --- a/app/dao/organization_dao.py +++ b/app/dao/organization_dao.py @@ -23,7 +23,6 @@ def dao_count_organizations_with_live_services(): Service.count_as_live.is_(True), ) ) - # TODO Need distinct here? return db.session.execute(stmt).scalar() or 0 From 9638e2b4f079f5fa91a924195c3cc0813dcb63e2 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 28 Oct 2024 13:20:51 -0700 Subject: [PATCH 128/128] code review feedback --- app/dao/service_permissions_dao.py | 9 - app/dao/service_sms_sender_dao.py | 9 +- app/dao/service_user_dao.py | 13 -- app/dao/services_dao.py | 255 +---------------------------- 4 files changed, 4 insertions(+), 282 deletions(-) diff --git a/app/dao/service_permissions_dao.py b/app/dao/service_permissions_dao.py index 95a40c903..0793b35b6 100644 --- a/app/dao/service_permissions_dao.py +++ b/app/dao/service_permissions_dao.py @@ -6,9 +6,6 @@ from app.models import ServicePermission def dao_fetch_service_permissions(service_id): - # return ServicePermission.query.filter( - # ServicePermission.service_id == service_id - # ).all() stmt = select(ServicePermission).filter(ServicePermission.service_id == service_id) return db.session.execute(stmt).scalars().all() @@ -21,12 +18,6 @@ def dao_add_service_permission(service_id, permission): def dao_remove_service_permission(service_id, permission): - # deleted = ServicePermission.query.filter( - # ServicePermission.service_id == service_id, - # ServicePermission.permission == permission, - # ).delete() - # db.session.commit() - # return deleted stmt = delete(ServicePermission).where( ServicePermission.service_id == service_id, diff --git a/app/dao/service_sms_sender_dao.py b/app/dao/service_sms_sender_dao.py index df0f2a3e9..82796b05f 100644 --- a/app/dao/service_sms_sender_dao.py +++ b/app/dao/service_sms_sender_dao.py @@ -17,9 +17,6 @@ def insert_service_sms_sender(service, sms_sender): def dao_get_service_sms_senders_by_id(service_id, service_sms_sender_id): - # return ServiceSmsSender.query.filter_by( - # id=service_sms_sender_id, service_id=service_id, archived=False - # ).one() stmt = select(ServiceSmsSender).filter_by( id=service_sms_sender_id, service_id=service_id, archived=False ) @@ -27,11 +24,7 @@ def dao_get_service_sms_senders_by_id(service_id, service_sms_sender_id): def dao_get_sms_senders_by_service_id(service_id): - # return ( - # ServiceSmsSender.query.filter_by(service_id=service_id, archived=False) - # .order_by(desc(ServiceSmsSender.is_default)) - # .all() - # ) + stmt = ( select(ServiceSmsSender) .filter_by(service_id=service_id, archived=False) diff --git a/app/dao/service_user_dao.py b/app/dao/service_user_dao.py index b02005a3f..d60c92ba6 100644 --- a/app/dao/service_user_dao.py +++ b/app/dao/service_user_dao.py @@ -6,24 +6,11 @@ from app.models import ServiceUser, User def dao_get_service_user(user_id, service_id): - # TODO: This has been changed to account for the test case failure - # that used this method but have any service user to return. Somehow, this - # started to throw an error with one() method in sqlalchemy 2.0 unlike 1.4 - # return ServiceUser.query.filter_by( - # user_id=user_id, service_id=service_id - # ).one_or_none() stmt = select(ServiceUser).filter_by(user_id=user_id, service_id=service_id) return db.session.execute(stmt).scalars().one_or_none() def dao_get_active_service_users(service_id): - # query = ( - # db.session.query(ServiceUser) - # .join(User, User.id == ServiceUser.user_id) - # .filter(User.state == "active", ServiceUser.service_id == service_id) - # ) - - # return query.all() stmt = ( select(ServiceUser) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 585fe83b1..1f8956865 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -85,88 +85,6 @@ def dao_count_live_services(): def dao_fetch_live_services_data(): year_start_date, year_end_date = get_current_calendar_year() - # most_recent_annual_billing = ( - # db.session.query( - # AnnualBilling.service_id, - # func.max(AnnualBilling.financial_year_start).label("year"), - # ) - # .group_by(AnnualBilling.service_id) - # .subquery() - # ) - - # this_year_ft_billing = FactBilling.query.filter( - # FactBilling.local_date >= year_start_date, - # FactBilling.local_date <= year_end_date, - # ).subquery() - - # data = ( - # db.session.query( - # Service.id.label("service_id"), - # Service.name.label("service_name"), - # Organization.name.label("organization_name"), - # Organization.organization_type.label("organization_type"), - # Service.consent_to_research.label("consent_to_research"), - # User.name.label("contact_name"), - # User.email_address.label("contact_email"), - # User.mobile_number.label("contact_mobile"), - # Service.go_live_at.label("live_date"), - # Service.volume_sms.label("sms_volume_intent"), - # Service.volume_email.label("email_volume_intent"), - # case( - # ( - # this_year_ft_billing.c.notification_type == NotificationType.EMAIL, - # func.sum(this_year_ft_billing.c.notifications_sent), - # ), - # else_=0, - # ).label("email_totals"), - # case( - # ( - # this_year_ft_billing.c.notification_type == NotificationType.SMS, - # func.sum(this_year_ft_billing.c.notifications_sent), - # ), - # else_=0, - # ).label("sms_totals"), - # AnnualBilling.free_sms_fragment_limit, - # ) - # .join(Service.annual_billing) - # .join( - # most_recent_annual_billing, - # and_( - # Service.id == most_recent_annual_billing.c.service_id, - # AnnualBilling.financial_year_start == most_recent_annual_billing.c.year, - # ), - # ) - # .outerjoin(Service.organization) - # .outerjoin( - # this_year_ft_billing, Service.id == this_year_ft_billing.c.service_id - # ) - # .outerjoin(User, Service.go_live_user_id == User.id) - # .filter( - # Service.count_as_live.is_(True), - # Service.active.is_(True), - # Service.restricted.is_(False), - # ) - # .group_by( - # Service.id, - # Organization.name, - # Organization.organization_type, - # Service.name, - # Service.consent_to_research, - # Service.count_as_live, - # Service.go_live_user_id, - # User.name, - # User.email_address, - # User.mobile_number, - # Service.go_live_at, - # Service.volume_sms, - # Service.volume_email, - # this_year_ft_billing.c.notification_type, - # AnnualBilling.free_sms_fragment_limit, - # ) - # .order_by(asc(Service.go_live_at)) - # .all() - # ) - most_recent_annual_billing = ( select( AnnualBilling.service_id, @@ -297,12 +215,6 @@ def dao_fetch_service_by_inbound_number(number): def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): - # query = Service.query.filter_by(id=service_id).options(joinedload(Service.api_keys)) - - # if only_active: - # query = query.filter(Service.active) - - # return query.one() stmt = ( select(Service).filter_by(id=service_id).options(joinedload(Service.api_keys)) ) @@ -312,16 +224,6 @@ def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): def dao_fetch_all_services_by_user(user_id, only_active=False): - # query = ( - # Service.query.filter(Service.users.any(id=user_id)) - # .order_by(asc(Service.created_at)) - # .options(joinedload(Service.users)) - # ) - - # if only_active: - # query = query.filter(Service.active) - - # return query.all() stmt = ( select(Service) @@ -336,12 +238,6 @@ def dao_fetch_all_services_by_user(user_id, only_active=False): def dao_fetch_all_services_created_by_user(user_id): - # query = Service.query.filter_by(created_by_id=user_id).order_by( - # asc(Service.created_at) - # ) - - # return query.all() - stmt = ( select(Service) .filter_by(created_by_id=user_id) @@ -358,16 +254,6 @@ def dao_fetch_all_services_created_by_user(user_id): VersionOptions(Template, history_class=TemplateHistory, must_write_history=False), ) def dao_archive_service(service_id): - # have to eager load templates and api keys so that we don't flush when we loop through them - # to ensure that db.session still contains the models when it comes to creating history objects - # service = ( - # Service.query.options( - # joinedload(Service.templates).subqueryload(Template.template_redacted), - # joinedload(Service.api_keys), - # ) - # .filter(Service.id == service_id) - # .one() - # ) stmt = ( select(Service) .options( @@ -392,11 +278,6 @@ def dao_archive_service(service_id): def dao_fetch_service_by_id_and_user(service_id, user_id): - # return ( - # Service.query.filter(Service.users.any(id=user_id), Service.id == service_id) - # .options(joinedload(Service.users)) - # .one() - # ) stmt = ( select(Service) @@ -508,11 +389,9 @@ def dao_remove_user_from_service(service, user): def delete_service_and_all_associated_db_objects(service): def _delete_commit(stmt): - # query.delete(synchronize_session=False) db.session.execute(stmt) db.session.commit() - # subq = db.session.query(Template.id).filter_by(service=service).subquery() subq = select(Template.id).filter_by(service=service).subquery() stmt = delete(TemplateRedacted).filter(TemplateRedacted.template_id.in_(subq)) @@ -553,23 +432,6 @@ def delete_service_and_all_associated_db_objects(service): def dao_fetch_todays_stats_for_service(service_id): today = utc_now().date() start_date = get_midnight_in_utc(today) - # return ( - # db.session.query( - # Notification.notification_type, - # Notification.status, - # func.count(Notification.id).label("count"), - # ) - # .filter( - # Notification.service_id == service_id, - # Notification.key_type != KeyType.TEST, - # Notification.created_at >= start_date, - # ) - # .group_by( - # Notification.notification_type, - # Notification.status, - # ) - # .all() - # ) stmt = ( select( Notification.notification_type, @@ -593,27 +455,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - # return ( - # db.session.query( - # NotificationAllTimeView.notification_type, - # NotificationAllTimeView.status, - # func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - # func.count(NotificationAllTimeView.id).label("count"), - # ) - # .filter( - # NotificationAllTimeView.service_id == service_id, - # NotificationAllTimeView.key_type != KeyType.TEST, - # NotificationAllTimeView.created_at >= start_date, - # NotificationAllTimeView.created_at < end_date, - # ) - # .group_by( - # NotificationAllTimeView.notification_type, - # NotificationAllTimeView.status, - # func.date_trunc("day", NotificationAllTimeView.created_at), - # ) - # .all() - # ) - stmt = ( select( NotificationAllTimeView.notification_type, @@ -642,27 +483,6 @@ def dao_fetch_stats_for_service_from_days_for_user( start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - # return ( - # db.session.query( - # NotificationAllTimeView.notification_type, - # NotificationAllTimeView.status, - # func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - # func.count(NotificationAllTimeView.id).label("count"), - # ) - # .filter( - # NotificationAllTimeView.service_id == service_id, - # NotificationAllTimeView.key_type != KeyType.TEST, - # NotificationAllTimeView.created_at >= start_date, - # NotificationAllTimeView.created_at < end_date, - # NotificationAllTimeView.created_by_id == user_id, - # ) - # .group_by( - # NotificationAllTimeView.notification_type, - # NotificationAllTimeView.status, - # func.date_trunc("day", NotificationAllTimeView.created_at), - # ) - # .all() - # ) stmt = ( select( NotificationAllTimeView.notification_type, @@ -741,15 +561,7 @@ def dao_fetch_todays_stats_for_all_services( VersionOptions(Service), ) def dao_suspend_service(service_id): - # have to eager load api keys so that we don't flush when we loop through them - # to ensure that db.session still contains the models when it comes to creating history objects - # service = ( - # Service.query.options( - # joinedload(Service.api_keys), - # ) - # .filter(Service.id == service_id) - # .one() - # ) + stmt = ( select(Service) .options(joinedload(Service.api_keys)) @@ -767,16 +579,12 @@ def dao_suspend_service(service_id): @autocommit @version_class(Service) def dao_resume_service(service_id): - # service = Service.query.get(service_id) service = db.session.get(Service, service_id) service.active = True def dao_fetch_active_users_for_service(service_id): - # query = User.query.filter(User.services.any(id=service_id), User.state == "active") - - # return query.all() stmt = select(User).where(User.services.any(id=service_id), User.state == "active") result = db.session.execute(stmt) @@ -784,27 +592,7 @@ def dao_fetch_active_users_for_service(service_id): def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500): - # return ( - # db.session.query( - # Notification.service_id.label("service_id"), - # func.count(Notification.id).label("notification_count"), - # ) - # .filter( - # Notification.service_id == Service.id, - # Notification.created_at >= start_date, - # Notification.created_at <= end_date, - # Notification.key_type != KeyType.TEST, - # Notification.notification_type == NotificationType.SMS, - # func.substr(Notification.normalised_to, 3, 7) == "7700900", - # Service.restricted == False, # noqa - # Service.active == True, # noqa - # ) - # .group_by( - # Notification.service_id, - # ) - # .having(func.count(Notification.id) > threshold) - # .all() - # ) + stmt = ( select( Notification.service_id.label("service_id"), @@ -884,23 +672,6 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 def get_live_services_with_organization(): - # query = ( - # db.session.query( - # Service.id.label("service_id"), - # Service.name.label("service_name"), - # Organization.id.label("organization_id"), - # Organization.name.label("organization_name"), - # ) - # .outerjoin(Service.organization) - # .filter( - # Service.count_as_live.is_(True), - # Service.active.is_(True), - # Service.restricted.is_(False), - # ) - # .order_by(Organization.name, Service.name) - # ) - - # return query.all() stmt = ( select( @@ -925,27 +696,7 @@ def get_live_services_with_organization(): def fetch_notification_stats_for_service_by_month_by_user( start_date, end_date, service_id, user_id ): - # return ( - # db.session.query( - # func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), - # NotificationAllTimeView.notification_type, - # (NotificationAllTimeView.status).label("notification_status"), - # func.count(NotificationAllTimeView.id).label("count"), - # ) - # .filter( - # NotificationAllTimeView.service_id == service_id, - # NotificationAllTimeView.created_at >= start_date, - # NotificationAllTimeView.created_at < end_date, - # NotificationAllTimeView.key_type != KeyType.TEST, - # NotificationAllTimeView.created_by_id == user_id, - # ) - # .group_by( - # func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), - # NotificationAllTimeView.notification_type, - # NotificationAllTimeView.status, - # ) - # .all() - # ) + stmt = ( select( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"),