Merge pull request #637 from GSA/notify-api-634

Create internal alarms/notifications for 50, 75, 90% of 10K db max

app/celery/scheduled_tasks.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 
 from flask import current_app
@@ -5,7 +6,7 @@ from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTick
 from sqlalchemy import between
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, zendesk_client
+from app import notify_celery, redis_store, zendesk_client
 from app.celery.tasks import (
     get_recipient_csv_and_template_and_sender_id,
     process_incomplete_jobs,
@@ -23,12 +24,16 @@ from app.dao.jobs_dao import (
     find_jobs_with_missing_rows,
     find_missing_row_for_job,
 )
-from app.dao.notifications_dao import notifications_not_yet_sent
+from app.dao.notifications_dao import (
+    dao_get_failed_notification_count,
+    notifications_not_yet_sent,
+)
 from app.dao.services_dao import (
     dao_find_services_sending_to_tv_numbers,
     dao_find_services_with_high_failure_rates,
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.delivery.send_to_providers import provider_to_use
 from app.models import (
     EMAIL_TYPE,
     JOB_STATUS_ERROR,
@@ -39,6 +44,8 @@ from app.models import (
 )
 from app.notifications.process_notifications import send_notification_to_queue
 
+MAX_NOTIFICATION_FAILS = 10000
+
 
 @notify_celery.task(name="run-scheduled-jobs")
 def run_scheduled_jobs():
@@ -91,6 +98,78 @@ def expire_or_delete_invitations():
         raise
 
 
+@notify_celery.task(name="check-db-notification-fails")
+def check_db_notification_fails():
+    """
+    We are going to use redis to keep track of the previous fail count.
+
+    If the number of fails is more than 100% of the limit, we want to send an alert every time this
+    runs, because it is urgent to fix it.
+
+    If the number is more than 25%, 50% or 75% of the limit, we only want to send an alert
+    on a breach.  I.e., if the last number was at 23% and the current number is 27%, send an email.
+    But if the last number was 26% and the current is 27%, don't.
+    """
+    last_value = redis_store.get("LAST_DB_NOTIFICATION_COUNT")
+    if not last_value:
+        last_value = 0
+
+    failed_count = dao_get_failed_notification_count()
+    if failed_count > last_value:
+        redis_store.set("LAST_DB_NOTIFICATION_COUNT", failed_count)
+    message = ""
+    curr_env = os.getenv("ENVIRONMENT")
+    if failed_count >= MAX_NOTIFICATION_FAILS:
+        message = f"We are over 100% in the db for failed notifications on {curr_env}"
+    elif (
+        failed_count >= MAX_NOTIFICATION_FAILS * 0.9
+        and last_value < MAX_NOTIFICATION_FAILS * 0.9
+    ):
+        message = (
+            "tts-notify-alerts@gsa.gov",
+            f"We crossed above 90% in the db for failed notifications on {curr_env}",
+        )
+
+    elif (
+        failed_count >= MAX_NOTIFICATION_FAILS * 0.75
+        and last_value < MAX_NOTIFICATION_FAILS * 0.75
+    ):
+        message = (
+            "tts-notify-alerts@gsa.gov",
+            f"We crossed above 75% in the db for failed notifications on {curr_env}",
+        )
+    elif (
+        failed_count >= MAX_NOTIFICATION_FAILS * 0.5
+        and last_value < MAX_NOTIFICATION_FAILS * 0.5
+    ):
+        message = (
+            "tts-notify-alerts@gsa.gov",
+            f"We crossed above 50% in the db for failed notifications on {curr_env}",
+        )
+    elif (
+        failed_count >= MAX_NOTIFICATION_FAILS * 0.25
+        and last_value < MAX_NOTIFICATION_FAILS * 0.25
+    ):
+        message = (
+            "tts-notify-alerts@gsa.gov",
+            f"We crossed above 25% in the db for failed notifications on {curr_env}",
+        )
+    # suppress any spam coming from development tier
+    if message and curr_env != "development":
+        provider = provider_to_use(EMAIL_TYPE, False)
+        from_address = '"{}" <{}@{}>'.format(
+            "Failed Notification Count Alert",
+            "test_sender",
+            current_app.config["NOTIFY_EMAIL_DOMAIN"],
+        )
+        provider.send_email(
+            from_address,
+            "tts-notify-alerts@gsa.gov",
+            "DB Notification Failures Level Breached",
+            body=str(message),
+        )
+
+
 @notify_celery.task(name="check-job-status")
 def check_job_status():
     """

app/config.py

@@ -199,6 +199,11 @@ class Config(object):
                 "schedule": timedelta(minutes=66),
                 "options": {"queue": QueueNames.PERIODIC},
             },
+            "check-db-notification-fails": {
+                "task": "check-db-notification-fails",
+                "schedule": crontab(minute="18, 48"),
+                "options": {"queue": QueueNames.PERIODIC},
+            },
             "check-job-status": {
                 "task": "check-job-status",
                 "schedule": crontab(),

app/dao/notifications_dao.py

@@ -20,6 +20,7 @@ from app.models import (
     EMAIL_TYPE,
     KEY_TYPE_TEST,
     NOTIFICATION_CREATED,
+    NOTIFICATION_FAILED,
     NOTIFICATION_PENDING,
     NOTIFICATION_PENDING_VIRUS_CHECK,
     NOTIFICATION_PERMANENT_FAILURE,
@@ -202,6 +203,11 @@ def dao_get_notification_count_for_service(*, service_id):
     return notification_count
 
 
+def dao_get_failed_notification_count():
+    failed_count = Notification.query.filter_by(status=NOTIFICATION_FAILED).count()
+    return failed_count
+
+
 def get_notification_with_personalisation(service_id, notification_id, key_type):
     filter_dict = {"service_id": service_id, "id": notification_id}
     if key_type:

poetry.lock

@@ -2168,16 +2168,6 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},

tests/app/celery/test_scheduled_tasks.py

@@ -8,6 +8,7 @@ from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTick
 
 from app.celery import scheduled_tasks
 from app.celery.scheduled_tasks import (
+    check_db_notification_fails,
     check_for_missing_rows_in_completed_jobs,
     check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
     check_job_status,
@@ -53,6 +54,51 @@ def test_should_call_expire_or_delete_invotations_on_expire_or_delete_invitation
     )
 
 
+def test_should_check_db_notification_fails_task_over_100_percent(
+    notify_db_session, mocker
+):
+    mock_dao = mocker.patch(
+        "app.celery.scheduled_tasks.dao_get_failed_notification_count"
+    )
+    mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use")
+    mock_dao.return_value = 100000
+    check_db_notification_fails()
+    assert mock_provider.call_count == 1
+
+
+def test_should_check_db_notification_fails_task_less_than_25_percent(
+    notify_db_session, mocker
+):
+    mock_dao = mocker.patch(
+        "app.celery.scheduled_tasks.dao_get_failed_notification_count"
+    )
+    mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use")
+    mock_dao.return_value = 10
+    check_db_notification_fails()
+    assert mock_provider.call_count == 0
+
+
+def test_should_check_db_notification_fails_task_over_50_percent(
+    notify_db_session, mocker
+):
+    # This tests that we only send an alert the 1st time we cross over 50%.  We don't want
+    # to be sending the same alert every hour, especially as it might be quite normal for the db
+    # fails to be at 25 or 50 for long periods of time.
+    mock_dao = mocker.patch(
+        "app.celery.scheduled_tasks.dao_get_failed_notification_count"
+    )
+    mock_provider = mocker.patch("app.celery.scheduled_tasks.provider_to_use")
+    mock_redis = mocker.patch("app.celery.scheduled_tasks.redis_store.get")
+    mock_dao.return_value = 5001
+    mock_redis.return_value = 0
+    check_db_notification_fails()
+    assert mock_provider.call_count == 1
+
+    mock_redis.return_value = 5001
+    check_db_notification_fails()
+    assert mock_provider.call_count == 1
+
+
 def test_should_update_scheduled_jobs_and_put_on_queue(mocker, sample_template):
     mocked = mocker.patch("app.celery.tasks.process_job.apply_async")