Undo recent changes related to Python upgrade

This changeset undoes the recent changes we tried after the Python 3.13 update as they had no bearing on the SSL cert validation errors.  Back to the drawing board!

Signed-off-by: Carlo Costino <carlo.costino@gsa.gov>

Makefile

@@ -70,7 +70,8 @@ run-celery: ## Run celery, TODO remove purge for staging/prod
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
-		--pool=solo
+		--pool=gevent
+		--concurrency=20
 
 
 .PHONY: dead-code

app/aws/s3.py

@@ -5,6 +5,7 @@ import time
 from io import StringIO
 
 import botocore
+import gevent
 from boto3 import Session
 from flask import current_app
 
@@ -249,7 +250,7 @@ def get_s3_files():
         for object_key in object_keys:
             read_s3_file(bucket_name, object_key, s3res)
             count = count + 1
-            time.sleep(0.2)
+            gevent.sleep(0.2)
     except Exception:
         current_app.logger.exception(
             f"Trouble reading {object_key} which is # {count} during cache regeneration"
@@ -410,7 +411,7 @@ def get_job_from_s3(service_id, job_id):
                 )
                 retries += 1
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
-                time.sleep(sleep_time)
+                gevent.sleep(sleep_time)
                 continue
             else:
                 # Typically this is "NoSuchKey"

app/celery/tasks.py

@@ -1,6 +1,6 @@
 import json
-import time
 
+import gevent
 from celery.signals import task_postrun
 from flask import current_app
 from requests import HTTPError, RequestException, request
@@ -84,7 +84,7 @@ def process_job(job_id, sender_id=None):
         process_row(row, template, job, service, sender_id=sender_id)
         count = count + 1
         if count % 3 == 0:
-            time.sleep(1)
+            gevent.sleep(1)
 
     # End point/Exit point for message send flow.
     job_complete(job, start=start)

gunicorn_config.py

@@ -7,7 +7,7 @@ import gunicorn  # noqa
 
 # This will give us a better stack trace if
 workers = 4
-worker_class = "sync"
+worker_class = "gevent"
 worker_connections = 256
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))
 statsd_host = "{}:8125".format(os.getenv("STATSD_HOST"))

gunicorn_entry.py

@@ -1,10 +1,5 @@
-# from gevent import monkey
+from gevent import monkey
 
-# commenting out while we disable gevent
+monkey.patch_all()
-# monkey.patch_all()
-
-import newrelic.agent  # noqa
-
-newrelic.agent.initialize("./newrelic.ini")
 
 from application import application  # noqa

manifest.yml

@@ -26,7 +26,7 @@ applications:
       - type: worker
         instances: ((worker_instances))
         memory: ((worker_memory))
-        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=solo --prefetch-multiplier=2
+        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=gevent --concurrency=20 --prefetch-multiplier=2
       - type: scheduler
         instances: 1
         memory: ((scheduler_memory))

scripts/migrate_and_run_web.sh

@@ -4,4 +4,4 @@ if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then
   flask db upgrade
 fi
 
-exec gunicorn -c ${HOME}/gunicorn_config.py gunicorn_entry:application
+exec newrelic-admin run-program gunicorn -c ${HOME}/gunicorn_config.py gunicorn_entry:application