diff --git a/Makefile b/Makefile index 59b1ae77d..70ce355a9 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,8 @@ run-celery: ## Run celery, TODO remove purge for staging/prod -A run_celery.notify_celery worker \ --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ - --pool=solo + --pool=gevent + --concurrency=20 .PHONY: dead-code diff --git a/app/aws/s3.py b/app/aws/s3.py index 54c9c85a3..9c397fc1a 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -5,6 +5,7 @@ import time from io import StringIO import botocore +import gevent from boto3 import Session from flask import current_app @@ -249,7 +250,7 @@ def get_s3_files(): for object_key in object_keys: read_s3_file(bucket_name, object_key, s3res) count = count + 1 - time.sleep(0.2) + gevent.sleep(0.2) except Exception: current_app.logger.exception( f"Trouble reading {object_key} which is # {count} during cache regeneration" @@ -410,7 +411,7 @@ def get_job_from_s3(service_id, job_id): ) retries += 1 sleep_time = backoff_factor * (2**retries) # Exponential backoff - time.sleep(sleep_time) + gevent.sleep(sleep_time) continue else: # Typically this is "NoSuchKey" diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 3a6a997b0..7cf3c0933 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,6 +1,6 @@ import json -import time +import gevent from celery.signals import task_postrun from flask import current_app from requests import HTTPError, RequestException, request @@ -84,7 +84,7 @@ def process_job(job_id, sender_id=None): process_row(row, template, job, service, sender_id=sender_id) count = count + 1 if count % 3 == 0: - time.sleep(1) + gevent.sleep(1) # End point/Exit point for message send flow. job_complete(job, start=start) diff --git a/gunicorn_config.py b/gunicorn_config.py index 55f946a11..fc50f4ffc 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -7,7 +7,7 @@ import gunicorn # noqa # This will give us a better stack trace if workers = 4 -worker_class = "sync" +worker_class = "gevent" worker_connections = 256 bind = "0.0.0.0:{}".format(os.getenv("PORT")) statsd_host = "{}:8125".format(os.getenv("STATSD_HOST")) diff --git a/gunicorn_entry.py b/gunicorn_entry.py index e36ab57c1..ab9a0d526 100644 --- a/gunicorn_entry.py +++ b/gunicorn_entry.py @@ -1,10 +1,5 @@ -# from gevent import monkey +from gevent import monkey -# commenting out while we disable gevent -# monkey.patch_all() - -import newrelic.agent # noqa - -newrelic.agent.initialize("./newrelic.ini") +monkey.patch_all() from application import application # noqa diff --git a/manifest.yml b/manifest.yml index 589c9b4a0..0c336e27a 100644 --- a/manifest.yml +++ b/manifest.yml @@ -26,7 +26,7 @@ applications: - type: worker instances: ((worker_instances)) memory: ((worker_memory)) - command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=solo --prefetch-multiplier=2 + command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --pool=gevent --concurrency=20 --prefetch-multiplier=2 - type: scheduler instances: 1 memory: ((scheduler_memory)) diff --git a/scripts/migrate_and_run_web.sh b/scripts/migrate_and_run_web.sh index 5f2006db3..ed999a24e 100755 --- a/scripts/migrate_and_run_web.sh +++ b/scripts/migrate_and_run_web.sh @@ -4,4 +4,4 @@ if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then flask db upgrade fi -exec gunicorn -c ${HOME}/gunicorn_config.py gunicorn_entry:application +exec newrelic-admin run-program gunicorn -c ${HOME}/gunicorn_config.py gunicorn_entry:application