create cronitor decorator that alerts if tasks fail

make a decorator that pings cronitor before and after each task run.
Designed for use with nightly tasks, so we have visibility if they
fail. We have a bunch of cronitor monitors set up - 5 character keys
that go into a URL that we then make a GET to with a self-explanatory
url path (run/fail/complete).

the cronitor URLs are defined in the credentials repo as a dictionary
of celery task names to URL slugs. If the name passed in to the
decorator  isn't in that dict, it won't run.

to use it, all you need to do is call `@cronitor(my_task_name)`
instead of `@notify_celery.task`, and make sure that the task name and
the matching slug are included in the credentials repo (or locally,
json dumped and stored in the CRONITOR_KEYS environment variable)

app/cronitor.py

@@ -0,0 +1,50 @@
+import requests
+from functools import wraps
+from flask import current_app
+
+
+def cronitor(task_name):
+    # check if task_name is in config
+    def decorator(func):
+        def ping_cronitor(command):
+            if not current_app.config['CRONITOR_ENABLED']:
+                return
+
+            task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
+            if not task_slug:
+                current_app.logger.error(
+                    'Cronitor enabled but task_name {} not found in environment'.format(task_name)
+                )
+
+            if command not in {'run', 'complete', 'fail'}:
+                raise ValueError('command {} not a valid cronitor command'.format(command))
+
+            resp = requests.get(
+                'https://cronitor.link/{}/{}'.format(task_slug, command),
+                # cronitor limits msg to 1000 characters
+                params={
+                    'host': current_app.config['API_HOST_NAME'],
+                }
+            )
+            if resp.status_code != 200:
+                current_app.logger.warning('Cronitor API returned {} for task {}, body {}'.format(
+                    resp.status_code,
+                    task_name,
+                    resp.text
+                ))
+
+        @wraps(func)
+        def inner_decorator(*args, **kwargs):
+            ping_cronitor('run')
+            try:
+                ret = func(*args, **kwargs)
+                status = 'complete'
+                return ret
+            except Exception:
+                status = 'fail'
+                raise
+            finally:
+                ping_cronitor(status)
+
+        return inner_decorator
+    return decorator