Merge pull request #1850 from GSA/system_certs

revert pip-system-certs

.github/dependabot.yml

@@ -13,5 +13,3 @@ updates:
       - "dependabot" # Custom label to identify Dependabot PRs
     assignees:
       - "alexjanousekGSA"
-    reviewers:
-      - "alexjanousekGSA"

Makefile

@@ -70,8 +70,8 @@ run-celery: ## Run celery, TODO remove purge for staging/prod
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
-		--pool=threads
-		--concurrency=10
+		--pool=eventlet
+		--concurrency=20
 
 
 .PHONY: dead-code

app/aws/s3.py

@@ -5,6 +5,7 @@ import time
 from io import StringIO
 
 import botocore
+import eventlet
 from boto3 import Session
 from flask import current_app
 
@@ -249,7 +250,7 @@ def get_s3_files():
         for object_key in object_keys:
             read_s3_file(bucket_name, object_key, s3res)
             count = count + 1
-            time.sleep(0.2)
+            eventlet.sleep(0.2)
     except Exception:
         current_app.logger.exception(
             f"Trouble reading {object_key} which is # {count} during cache regeneration"
@@ -410,7 +411,7 @@ def get_job_from_s3(service_id, job_id):
                 )
                 retries += 1
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
-                time.sleep(sleep_time)
+                eventlet.sleep(sleep_time)
                 continue
             else:
                 # Typically this is "NoSuchKey"

app/celery/nightly_tasks.py

@@ -73,10 +73,12 @@ def delete_notifications_older_than_retention():
     )
 
 
+@notify_celery.task(name="delete-sms-notifications-older-than-retention")
 def delete_sms_notifications_older_than_retention():
     _delete_notifications_older_than_retention_by_type(NotificationType.SMS)
 
 
+@notify_celery.task(name="delete-email-notifications-older-than-retention")
 def delete_email_notifications_older_than_retention():
     _delete_notifications_older_than_retention_by_type(NotificationType.EMAIL)
 

app/celery/tasks.py

@@ -1,6 +1,6 @@
 import json
-import time
 
+import eventlet
 from celery.signals import task_postrun
 from flask import current_app
 from requests import HTTPError, RequestException, request
@@ -84,7 +84,7 @@ def process_job(job_id, sender_id=None):
         process_row(row, template, job, service, sender_id=sender_id)
         count = count + 1
         if count % 3 == 0:
-            time.sleep(1)
+            eventlet.sleep(1)
 
     # End point/Exit point for message send flow.
     job_complete(job, start=start)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -151,7 +151,7 @@ class AwsCloudwatchClient(Client):
     #         result = temp_client.get_query_results(queryId=query_id)
     #         if result['status'] == 'Complete':
     #             break
-    #         time.sleep(1)
+    #         eventlet.sleep(1)
 
     #     delivery_receipts = []
     #     for log in result['results']:

app/template/rest.py

@@ -189,8 +189,9 @@ def preview_template_by_id_and_service_id(service_id, template_id):
             },
             status_code=400,
         )
-
-    data["subject"] = template_object.subject
+    # Only emails have subjects, SMS do not
+    if hasattr(template_object, "subject"):
+        data["subject"] = template_object.subject
     data["content"] = template_object.content_with_placeholders_filled_in
 
     return jsonify(data)

app/user/rest.py

@@ -89,6 +89,7 @@ def handle_integrity_error(exc):
 
 @user_blueprint.route("", methods=["POST"])
 def create_user():
+    current_app.logger.info("#invites: enter create_user()")
     req_json = request.get_json()
     user_to_create = create_user_schema.load(req_json)
 
@@ -96,6 +97,7 @@ def create_user():
         user_to_create, password=req_json.get("password"), validated_email_access=True
     )
     result = user_to_create.serialize()
+    current_app.logger.info("#invites: returning the new user")
     return jsonify(data=result), 201
 
 
@@ -597,14 +599,22 @@ def set_permissions(user_id, service_id):
 
 @user_blueprint.route("/get-login-gov-user", methods=["POST"])
 def get_user_login_gov_user():
+    current_app.logger.info("#invites: enter get_user_login_gov_user")
     request_args = request.get_json()
     login_uuid = request_args["login_uuid"]
     email = request_args["email"]
+    current_app.logger.info(
+        f"#invites: login_uuid {login_uuid} email is null? {email is None}"
+    )
+
     user = get_login_gov_user(login_uuid, email)
 
     if user is None:
+        current_app.logger.info("#invites: couldn't find user, returning empty data")
         return jsonify({})
+    current_app.logger.info("#invites: found the user, serializing")
     result = user.serialize()
+    current_app.logger.info("#invites: returning successfully")
     return jsonify(data=result)
 
 
@@ -615,9 +625,7 @@ def fetch_user_by_email():
             hilite(f"enter fetch_user_by_email with {request.get_json()}")
         )
         email = email_data_request_schema.load(request.get_json())
-        debug_not_production(hilite(f"request schema loads {email}"))
         fetched_user = get_user_by_email(email["email"])
-        debug_not_production(hilite(f"fetched user is {fetched_user}"))
         result = fetched_user.serialize()
         return jsonify(data=result)
     except Exception as e:

gunicorn_config.py

@@ -1,11 +1,17 @@
-import os
-import socket
-import sys
-import traceback
-
 import eventlet
-import gunicorn
 
+eventlet.monkey_patch()  # this has to be called before other imports or monkey patching doesn't happen
+
+import os  # noqa
+import socket  # noqa
+import sys  # noqa
+import traceback  # noqa
+
+import eventlet.debug as debug  # noqa
+import gunicorn  # noqa
+
+# This will give us a better stack trace if blocking occurs (if True)
+debug.hub_blocking_detection(False)
 workers = 4
 worker_class = "eventlet"
 worker_connections = 256

manifest.yml

@@ -26,7 +26,7 @@ applications:
       - type: worker
         instances: ((worker_instances))
         memory: ((worker_memory))
-        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=threads --concurrency=10 --prefetch-multiplier=2
+        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=eventlet --concurrency=20 --prefetch-multiplier=2
       - type: scheduler
         instances: 1
         memory: ((scheduler_memory))

notifications_utils/logging.py

@@ -179,7 +179,7 @@ class JSONFormatter(BaseJSONFormatter):
             # for example, delivery receipts, where the formatting fails
             # This is not a huge problem, don't dump stack traces into the logs
             # for it.
-            logger.warning(f"failed to format log message: {e}")
+            logger.debug(f"failed to format log message: {e}")
         except IndexError as e:
-            logger.exception(f"failed to format log message: {e}")
+            logger.debug(f"failed to format log message: {e}")
         return log_record

pyproject.toml

@@ -34,7 +34,7 @@ flask-redis = "==0.4.0"
 flask-sqlalchemy = "^3.1.1"
 gunicorn = {version = "==23.0.0", extras = ["eventlet"]}
 iso8601 = "==2.1.0"
-jsonschema = {version = "==4.24.0", extras = ["format"]}
+jsonschema = {version = "==4.25.0", extras = ["format"]}
 lxml = "==6.0.0"
 marshmallow = "^4.0.0"
 marshmallow-sqlalchemy = "^1.4.2"
@@ -46,13 +46,13 @@ pyjwt = "==2.10.1"
 python-dotenv = "==1.1.1"
 sqlalchemy = "==2.0.41"
 werkzeug = "^3.0.6"
-faker = "^37.4.0"
+faker = "^37.4.2"
 async-timeout = "^5.0.1"
 bleach = "^6.1.0"
 geojson = "^3.2.0"
 numpy = "^2.3.1"
 ordered-set = "^4.1.0"
-phonenumbers = "^9.0.9"
+phonenumbers = "^9.0.10"
 python-json-logger = "^3.3.0"
 regex = "^2024.11.6"
 shapely = "^2.1.1"
@@ -66,7 +66,7 @@ markupsafe = "^3.0.2"
 pycparser = "^2.22"
 python-dateutil = "^2.9.0.post0"
 pyyaml = "^6.0.2"
-s3transfer = "^0.13.0"
+s3transfer = "^0.13.1"
 six = "^1.16.0"
 urllib3 = "^2.5.0"
 webencodings = "^0.5.1"
@@ -75,7 +75,7 @@ jinja2 = "^3.1.6"
 redis = "^5.0.8"
 requests = "^2.32.4"
 flask-socketio = "^5.5.1"
-virtualenv = "^20.31.2"
+virtualenv = "^20.32.0"
 marshmallow-enum = "^1.5.1"
 awscli = "^1.40.36"
 typing-extensions = "^4.14.1"
@@ -87,12 +87,12 @@ pytest = "^8.4.0"
 awscli = "^1.33.32"
 bandit = "*"
 black = "^25.1.0"
-cyclonedx-python-lib = "^10.4.1"
+cyclonedx-python-lib = "^11.0.0"
 cloudfoundry-client = "*"
 exceptiongroup = "==1.3.0"
 flake8 = "^7.3.0"
 flake8-bugbear = "^24.12.12"
-freezegun = "^1.5.2"
+freezegun = "^1.5.3"
 honcho = "*"
 isort = "^6.0.1"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}