diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 1742a310c..2c4d31d8c 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -1,4 +1,3 @@ -import os from datetime import datetime, timedelta from flask import current_app @@ -6,7 +5,7 @@ from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTick from sqlalchemy import between from sqlalchemy.exc import SQLAlchemyError -from app import notify_celery, redis_store, zendesk_client +from app import notify_celery, zendesk_client from app.celery.tasks import ( get_recipient_csv_and_template_and_sender_id, process_incomplete_jobs, @@ -24,16 +23,12 @@ from app.dao.jobs_dao import ( find_jobs_with_missing_rows, find_missing_row_for_job, ) -from app.dao.notifications_dao import ( - dao_get_failed_notification_count, - notifications_not_yet_sent, -) +from app.dao.notifications_dao import notifications_not_yet_sent from app.dao.services_dao import ( dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates, ) from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago -from app.delivery.send_to_providers import provider_to_use from app.enums import JobStatus, NotificationType from app.models import Job from app.notifications.process_notifications import send_notification_to_queue @@ -92,82 +87,6 @@ def expire_or_delete_invitations(): raise -# TODO THIS IS ACTUALLY DEPRECATED, WE ARE REMOVING PHONE NUMBERS FROM THE DB -# SO THERE WILL BE NO REASON TO KEEP TRACK OF THIS COUNT -@notify_celery.task(name="check-db-notification-fails") -def check_db_notification_fails(): - """ - We are going to use redis to keep track of the previous fail count. - - If the number of fails is more than 100% of the limit, we want to send an alert every time this - runs, because it is urgent to fix it. - - If the number is more than 25%, 50% or 75% of the limit, we only want to send an alert - on a breach. I.e., if the last number was at 23% and the current number is 27%, send an email. - But if the last number was 26% and the current is 27%, don't. - """ - last_value = redis_store.get("LAST_DB_NOTIFICATION_COUNT") - if not last_value: - last_value = 0 - else: - last_value = int(last_value.decode("utf-8")) - - failed_count = dao_get_failed_notification_count() - if failed_count > last_value: - redis_store.set("LAST_DB_NOTIFICATION_COUNT", failed_count) - message = "" - curr_env = os.getenv("ENVIRONMENT") - if failed_count >= MAX_NOTIFICATION_FAILS: - message = f"We are over 100% in the db for failed notifications on {curr_env}" - elif ( - failed_count >= MAX_NOTIFICATION_FAILS * 0.9 - and last_value < MAX_NOTIFICATION_FAILS * 0.9 - ): - message = ( - "tts-notify-alerts@gsa.gov", - f"We crossed above 90% in the db for failed notifications on {curr_env}", - ) - - elif ( - failed_count >= MAX_NOTIFICATION_FAILS * 0.75 - and last_value < MAX_NOTIFICATION_FAILS * 0.75 - ): - message = ( - "tts-notify-alerts@gsa.gov", - f"We crossed above 75% in the db for failed notifications on {curr_env}", - ) - elif ( - failed_count >= MAX_NOTIFICATION_FAILS * 0.5 - and last_value < MAX_NOTIFICATION_FAILS * 0.5 - ): - message = ( - "tts-notify-alerts@gsa.gov", - f"We crossed above 50% in the db for failed notifications on {curr_env}", - ) - elif ( - failed_count >= MAX_NOTIFICATION_FAILS * 0.25 - and last_value < MAX_NOTIFICATION_FAILS * 0.25 - ): - message = ( - "tts-notify-alerts@gsa.gov", - f"We crossed above 25% in the db for failed notifications on {curr_env}", - ) - # suppress any spam coming from development tier - if message and curr_env != "development": - provider = provider_to_use(NotificationType.EMAIL, False) - from_address = '"{}" <{}@{}>'.format( - "Failed Notification Count Alert", - "test_sender", - current_app.config["NOTIFY_EMAIL_DOMAIN"], - ) - provider.send_email( - from_address, - "tts-notify-alerts@gsa.gov", - "DB Notification Failures Level Breached", - body=str(message), - ) - - @notify_celery.task(name="check-job-status") def check_job_status(): """ diff --git a/poetry.lock b/poetry.lock index 8c2e521c6..d2b32cc39 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2488,6 +2488,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -3529,7 +3530,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 94b586a3a..1652700f0 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -8,7 +8,6 @@ from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTick from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( - check_db_notification_fails, check_for_missing_rows_in_completed_jobs, check_for_services_with_high_failure_rates_or_sending_to_tv_numbers, check_job_status, @@ -49,53 +48,6 @@ def test_should_call_expire_or_delete_invotations_on_expire_or_delete_invitation ) -def test_should_check_db_notification_fails_task_over_100_percent( - notify_db_session, mocker -): - mock_dao = mocker.patch( - "app.celery.scheduled_tasks.dao_get_failed_notification_count" - ) - mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use") - mock_dao.return_value = 100000 - check_db_notification_fails() - assert mock_provider.call_count == 1 - - -def test_should_check_db_notification_fails_task_less_than_25_percent( - notify_db_session, mocker -): - mock_dao = mocker.patch( - "app.celery.scheduled_tasks.dao_get_failed_notification_count" - ) - mock_redis = mocker.patch("app.celery.scheduled_tasks.redis_store") - mock_redis.get.return_value = 0 - mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use") - mock_dao.return_value = 10 - check_db_notification_fails() - assert mock_provider.call_count == 0 - - -def test_should_check_db_notification_fails_task_over_50_percent( - notify_db_session, mocker -): - # This tests that we only send an alert the 1st time we cross over 50%. We don't want - # to be sending the same alert every hour, especially as it might be quite normal for the db - # fails to be at 25 or 50 for long periods of time. - mock_dao = mocker.patch( - "app.celery.scheduled_tasks.dao_get_failed_notification_count" - ) - mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use") - mock_redis = mocker.patch("app.celery.scheduled_tasks.redis_store") - mock_dao.return_value = 5001 - mock_redis.get.return_value = "0".encode("utf-8") - check_db_notification_fails() - assert mock_provider.call_count == 1 - - mock_redis.get.return_value = "5001".encode("utf-8") - check_db_notification_fails() - assert mock_provider.call_count == 1 - - def test_should_update_scheduled_jobs_and_put_on_queue(mocker, sample_template): mocked = mocker.patch("app.celery.tasks.process_job.apply_async") diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 74d90aaaf..21182972d 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -200,10 +200,10 @@ def test_send_user_sms_code(client, sample_user, sms_code_template, mocker): """ notify_service = dao_fetch_service_by_id(current_app.config["NOTIFY_SERVICE_ID"]) - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.raw_set") auth_header = create_admin_authorization_header() mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -241,10 +241,10 @@ def test_send_user_code_for_sms_with_optional_to_field( Tests POST endpoint /user//sms-code with optional to field """ - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.raw_set") to_number = "+447119876757" mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111") mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async") @@ -468,10 +468,10 @@ def test_send_user_email_code( deliver_email = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") sample_user.auth_type = auth_type - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.raw_set") admin_request.post( "user.send_user_2fa_code", @@ -584,10 +584,10 @@ def test_user_verify_email_code_fails_if_code_already_used( def test_send_user_2fa_code_sends_from_number_for_international_numbers( client, sample_user, mocker, sms_code_template ): - mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get") + mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get") mock_redis_get.return_value = "foo" - mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set") + mocker.patch("app.user.rest.redis_store.raw_set") sample_user.mobile_number = "+601117224412" auth_header = create_admin_authorization_header()