From deaaa259f4ab6e010210ffcfec4e3b58f9d04a1d Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Mon, 3 Apr 2017 11:16:20 +0100 Subject: [PATCH 1/2] Bump the memory on delivery workers on prod to try and avoid memory issues which can cause delivery worker to fail. --- manifest-delivery-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml index 53c8d2f12..f6f27d21a 100644 --- a/manifest-delivery-production.yml +++ b/manifest-delivery-production.yml @@ -3,4 +3,4 @@ inherit: manifest-delivery-base.yml instances: 2 -memory: 1G +memory: 2G From 8f5e1318eb9ead04f4e1a098f22f8e67d72cf2d1 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Mon, 3 Apr 2017 11:17:27 +0100 Subject: [PATCH 2/2] Spread out the scheduled jobs. Before the longer tasks all ran on the hour, at midnight, 1 and 2. Now we run one at midnight, one at quarter past, half past then at quarter too. Trying to spread the load. --- app/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/config.py b/app/config.py index cf1288cbe..619532d3d 100644 --- a/app/config.py +++ b/app/config.py @@ -119,7 +119,7 @@ class Config(object): }, 'delete-failed-notifications': { 'task': 'delete-failed-notifications', - 'schedule': crontab(minute=0, hour='0,1,2'), + 'schedule': crontab(minute=15, hour='0,1,2'), 'options': {'queue': 'periodic'} }, 'delete-successful-notifications': { @@ -139,12 +139,12 @@ class Config(object): }, 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', - 'schedule': crontab(minute=0, hour='0,1,2'), + 'schedule': crontab(minute=30, hour='0,1,2'), 'options': {'queue': 'periodic'} }, 'remove_csv_files': { 'task': 'remove_csv_files', - 'schedule': crontab(minute=1, hour='0,1,2'), + 'schedule': crontab(minute=45, hour='0,1,2'), 'options': {'queue': 'periodic'} } }