From ad8cf3f3a6ba94bfc4503b7ddf5b1a120fd9c6c2 Mon Sep 17 00:00:00 2001 From: sakisv Date: Thu, 23 Dec 2021 19:28:17 +0200 Subject: [PATCH] Reduce pool size from 30 to 15 connections Having a pool size of 30 connections means that if we receive a big number of requests, with the current configuration, the API would end up holding onto 30 connections per worker * 4 workers per instance * 35 instances = 4200 connections. With a limit of 5000 connections, this means that we would only have 800 connections to share between the workers or for overflow usage (btw, even the overflow for the API would take us above the 5000 limit - 10 overflow connections per worker * 4 * 35 = 1400 connections, total 5600 _only_ for the API). During our load tests this led to a deadlock situation where nothing could retrieve connections to deal with a queue build-up. The reduced pool size allowed for a much more graceful degradation of the service where, after significant load we would increase the response times but still manage to serve all the requests. --- manifest.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 09c0a5551..80d46db8d 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -2,7 +2,7 @@ 'notify-api': { 'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', - 'sqlalchemy_pool_size': 30, + 'sqlalchemy_pool_size': 15, 'additional_env_vars': { 'STATSD_HOST': None },