Merge pull request #736 from GSA/notify-api-714

add stats for cache hits/misses and remove debug message

app/aws/s3.py

@@ -5,6 +5,7 @@ from boto3 import Session
 from expiringdict import ExpiringDict
 from flask import current_app
 
+from app import redis_store
 from app.clients import AWS_CLIENT_CONFIG
 
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
@@ -12,6 +13,9 @@ FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
 
 JOBS = ExpiringDict(max_len=100, max_age_seconds=3600 * 4)
 
+JOBS_CACHE_HITS = "JOBS_CACHE_HITS"
+JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES"
+
 
 def get_s3_file(bucket_name, file_location, access_key, secret_key, region):
     s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
@@ -72,6 +76,26 @@ def get_job_from_s3(service_id, job_id):
     return obj.get()["Body"].read().decode("utf-8")
 
 
+def incr_jobs_cache_misses():
+    if not redis_store.get(JOBS_CACHE_MISSES):
+        redis_store.set(JOBS_CACHE_MISSES, 1)
+    else:
+        redis_store.incr(JOBS_CACHE_MISSES)
+    hits = redis_store.get(JOBS_CACHE_HITS).decode("utf-8")
+    misses = redis_store.get(JOBS_CACHE_MISSES).decode("utf-8")
+    current_app.logger.info(f"JOBS CACHE MISS hits {hits} misses {misses}")
+
+
+def incr_jobs_cache_hits():
+    if not redis_store.get(JOBS_CACHE_HITS):
+        redis_store.set(JOBS_CACHE_HITS, 1)
+    else:
+        redis_store.incr(JOBS_CACHE_HITS)
+    hits = redis_store.get(JOBS_CACHE_HITS).decode("utf-8")
+    misses = redis_store.get(JOBS_CACHE_MISSES).decode("utf-8")
+    current_app.logger.info(f"JOBS CACHE MISS hits {hits} misses {misses}")
+
+
 def get_phone_number_from_s3(service_id, job_id, job_row_number):
     # We don't want to constantly pull down a job from s3 every time we need a phone number.
     # At the same time we don't want to store it in redis or the db
@@ -80,6 +104,10 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
     if job is None:
         job = get_job_from_s3(service_id, job_id)
         JOBS[job_id] = job
+        incr_jobs_cache_misses()
+    else:
+        incr_jobs_cache_hits()
+
     job = job.split("\r\n")
     first_row = job[0]
     job.pop(0)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -107,7 +107,6 @@ class AwsCloudwatchClient(Client):
         if all_log_events and len(all_log_events) > 0:
             event = all_log_events[0]
             message = json.loads(event["message"])
-            current_app.logger.info(f"MESSAGE {message}")
             return (
                 "success",
                 message["delivery"]["providerResponse"],

tests/app/aws/test_s3.py

@@ -66,6 +66,7 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
 def test_get_phone_number_from_s3(
     mocker, job, job_id, job_row_number, expected_phone_number
 ):
+    mocker.patch("app.aws.s3.redis_store")
     get_job_mock = mocker.patch("app.aws.s3.get_job_from_s3")
     get_job_mock.return_value = job
     phone_number = get_phone_number_from_s3("service_id", job_id, job_row_number)