From 1eeceb072eba1199c0ba48476ae05fc02fa9bbe1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 29 Jul 2025 10:22:36 -0700 Subject: [PATCH 1/3] revert newrelic changes and disable gevent --- Makefile | 2 +- app/aws/s3.py | 5 ++--- app/celery/tasks.py | 3 +-- app/clients/cloudwatch/aws_cloudwatch.py | 2 +- gunicorn_config.py | 2 +- gunicorn_entry.py | 5 +++-- manifest.yml | 2 +- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 70ce355a9..4d861cf77 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ run-celery: ## Run celery, TODO remove purge for staging/prod -A run_celery.notify_celery worker \ --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ - --pool=gevent + --pool=solo --concurrency=20 diff --git a/app/aws/s3.py b/app/aws/s3.py index 9c397fc1a..54c9c85a3 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -5,7 +5,6 @@ import time from io import StringIO import botocore -import gevent from boto3 import Session from flask import current_app @@ -250,7 +249,7 @@ def get_s3_files(): for object_key in object_keys: read_s3_file(bucket_name, object_key, s3res) count = count + 1 - gevent.sleep(0.2) + time.sleep(0.2) except Exception: current_app.logger.exception( f"Trouble reading {object_key} which is # {count} during cache regeneration" @@ -411,7 +410,7 @@ def get_job_from_s3(service_id, job_id): ) retries += 1 sleep_time = backoff_factor * (2**retries) # Exponential backoff - gevent.sleep(sleep_time) + time.sleep(sleep_time) continue else: # Typically this is "NoSuchKey" diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 7cf3c0933..43c2a0cc7 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,6 +1,5 @@ import json -import gevent from celery.signals import task_postrun from flask import current_app from requests import HTTPError, RequestException, request @@ -84,7 +83,7 @@ def process_job(job_id, sender_id=None): process_row(row, template, job, service, sender_id=sender_id) count = count + 1 if count % 3 == 0: - gevent.sleep(1) + time.sleep(1) # End point/Exit point for message send flow. job_complete(job, start=start) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 3c111c34a..0a6d3d7be 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -151,7 +151,7 @@ class AwsCloudwatchClient(Client): # result = temp_client.get_query_results(queryId=query_id) # if result['status'] == 'Complete': # break - # gevent.sleep(1) + # time.sleep(1) # delivery_receipts = [] # for log in result['results']: diff --git a/gunicorn_config.py b/gunicorn_config.py index fc50f4ffc..55f946a11 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -7,7 +7,7 @@ import gunicorn # noqa # This will give us a better stack trace if workers = 4 -worker_class = "gevent" +worker_class = "sync" worker_connections = 256 bind = "0.0.0.0:{}".format(os.getenv("PORT")) statsd_host = "{}:8125".format(os.getenv("STATSD_HOST")) diff --git a/gunicorn_entry.py b/gunicorn_entry.py index ab9a0d526..cd7374238 100644 --- a/gunicorn_entry.py +++ b/gunicorn_entry.py @@ -1,5 +1,6 @@ -from gevent import monkey +# from gevent import monkey -monkey.patch_all() +# commenting out while we disable gevent +# monkey.patch_all() from application import application # noqa diff --git a/manifest.yml b/manifest.yml index 0c336e27a..b21727940 100644 --- a/manifest.yml +++ b/manifest.yml @@ -26,7 +26,7 @@ applications: - type: worker instances: ((worker_instances)) memory: ((worker_memory)) - command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=gevent --concurrency=20 --prefetch-multiplier=2 + command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=solo --concurrency=20 --prefetch-multiplier=2 - type: scheduler instances: 1 memory: ((scheduler_memory)) From d0092d4bf65c470b3e45dc074470b86bd983efe5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 29 Jul 2025 10:33:29 -0700 Subject: [PATCH 2/3] fix --- app/celery/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 43c2a0cc7..3a6a997b0 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,4 +1,5 @@ import json +import time from celery.signals import task_postrun from flask import current_app From f5b4b83097914c20dde17607306dac139b6e1661 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 29 Jul 2025 11:09:08 -0700 Subject: [PATCH 3/3] remove concurrency for now --- Makefile | 1 - manifest.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4d861cf77..59b1ae77d 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,6 @@ run-celery: ## Run celery, TODO remove purge for staging/prod --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ --pool=solo - --concurrency=20 .PHONY: dead-code diff --git a/manifest.yml b/manifest.yml index b21727940..589c9b4a0 100644 --- a/manifest.yml +++ b/manifest.yml @@ -26,7 +26,7 @@ applications: - type: worker instances: ((worker_instances)) memory: ((worker_memory)) - command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=solo --concurrency=20 --prefetch-multiplier=2 + command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=solo --prefetch-multiplier=2 - type: scheduler instances: 1 memory: ((scheduler_memory))