Refactor statsd logging

Removed all existing statsd logging and replaced with:

- statsd decorator. Infers the stat name from the decorated function call. Delegates statsd call to statsd client. Calls incr and timing for each decorated method. This is applied to all tasks and all dao methods that touch the notifications/notification_history tables

- statsd client changed to prefix all stats with "notification.api."

- Relies on https://github.com/alphagov/notifications-utils/pull/61 for request logging. Once integrated we pass the statsd client to the logger, allowing us to statsd all API calls. This passes in the start time and the method to be called (NOT the url) onto the global flask object. We then construct statsd counters and timers in the following way

	notifications.api.POST.notifications.send_notification.200

This should allow us to aggregate to the level of

	- API or ADMIN
	- POST or GET etc
	- modules
	- methods
	- status codes

Finally we count the callbacks received from 3rd parties to mapped status.

app/dao/notifications_dao.py

@@ -31,9 +31,10 @@ from app.clients import (
     STATISTICS_REQUESTED
 )
 from app.dao.dao_utils import transactional
-from app.statsd_decorators import statsd_timer
+from app.statsd_decorators import statsd
 
-@statsd_timer(namespace="dao")
+
+@statsd(namespace="dao")
 def dao_get_notification_statistics_for_service(service_id, limit_days=None):
     query_filter = [NotificationStatistics.service_id == service_id]
     if limit_days is not None:
@@ -45,7 +46,7 @@ def dao_get_notification_statistics_for_service(service_id, limit_days=None):
     ).all()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
     return NotificationStatistics.query.filter_by(
         service_id=service_id,
@@ -53,12 +54,12 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day):
     ).order_by(desc(NotificationStatistics.day)).first()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_notification_statistics_for_day(day):
     return NotificationStatistics.query.filter_by(day=day).all()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_potential_notification_statistics_for_day(day):
     all_services = db.session.query(
         Service.id,
@@ -106,7 +107,7 @@ def create_notification_statistics_dict(service_id, day):
     }
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_7_day_agg_notification_statistics_for_service(service_id,
                                                           date_from,
                                                           week_count=52):
@@ -134,7 +135,7 @@ def dao_get_7_day_agg_notification_statistics_for_service(service_id,
     )
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_template_statistics_for_service(service_id, limit_days=None):
     query_filter = [TemplateStatistics.service_id == service_id]
     if limit_days is not None:
@@ -143,7 +144,7 @@ def dao_get_template_statistics_for_service(service_id, limit_days=None):
         desc(TemplateStatistics.updated_at)).all()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_get_template_statistics_for_template(template_id):
     return TemplateStatistics.query.filter(
         TemplateStatistics.template_id == template_id
@@ -152,7 +153,7 @@ def dao_get_template_statistics_for_template(template_id):
     ).all()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 @transactional
 def dao_create_notification(notification, notification_type):
     if notification.job_id:
@@ -260,7 +261,7 @@ def _update_notification_status(notification, status, notification_statistics_st
     return True
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 @transactional
 def update_notification_status_by_id(notification_id, status, notification_statistics_status=None):
     notification = Notification.query.with_lockmode("update").filter(
@@ -279,7 +280,7 @@ def update_notification_status_by_id(notification_id, status, notification_stati
     )
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 @transactional
 def update_notification_status_by_reference(reference, status, notification_statistics_status):
     notification = Notification.query.filter(Notification.reference == reference,
@@ -296,7 +297,7 @@ def update_notification_status_by_reference(reference, status, notification_stat
     )
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def dao_update_notification(notification):
     notification.updated_at = datetime.utcnow()
     notification_history = NotificationHistory.query.get(notification.id)
@@ -305,7 +306,7 @@ def dao_update_notification(notification):
     db.session.commit()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 @transactional
 def update_provider_stats(
         id_,
@@ -340,12 +341,12 @@ def update_provider_stats(
         db.session.add(provider_stats)
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def get_notification_for_job(service_id, job_id, notification_id):
     return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -357,7 +358,7 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
     )
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def get_notification(service_id, notification_id, key_type=None):
     filter_dict = {'service_id': service_id, 'id': notification_id}
     if key_type:
@@ -366,7 +367,7 @@ def get_notification(service_id, notification_id, key_type=None):
     return Notification.query.filter_by(**filter_dict).one()
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def get_notification_by_id(notification_id):
     return Notification.query.filter_by(id=notification_id).first()
 
@@ -375,7 +376,7 @@ def get_notifications(filter_dict=None):
     return _filter_query(Notification.query, filter_dict=filter_dict)
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def get_notifications_for_service(service_id,
                                   filter_dict=None,
                                   page=1,
@@ -415,7 +416,7 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
-@statsd_timer(namespace="dao")
+@statsd(namespace="dao")
 def delete_notifications_created_more_than_a_week_ago(status):
     seven_days_ago = date.today() - timedelta(days=7)
     deleted = db.session.query(Notification).filter(

app/dao/services_dao.py

@@ -24,6 +24,7 @@ from app.models import (
     InvitedUser,
     Service
 )
+from app.statsd_decorators import statsd
 
 
 def dao_fetch_all_services():
@@ -136,10 +137,12 @@ def delete_service_and_all_associated_db_objects(service):
     db.session.commit()
 
 
+@statsd(namespace="dao")
 def dao_fetch_stats_for_service(service_id):
     return _stats_for_service_query(service_id).all()
 
 
+@statsd(namespace="dao")
 def dao_fetch_todays_stats_for_service(service_id):
     return _stats_for_service_query(service_id).filter(
         func.date(Notification.created_at) == date.today()
@@ -159,6 +162,7 @@ def _stats_for_service_query(service_id):
     )
 
 
+@statsd(namespace="dao")
 def dao_fetch_weekly_historical_stats_for_service(service_id):
     monday_of_notification_week = func.date_trunc('week', NotificationHistory.created_at).label('week_start')
     return db.session.query(