From d855b4e4ecc0d902be5387ff57c78d9da4f89834 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Tue, 6 Feb 2018 09:35:33 +0000 Subject: [PATCH] Removed statsd from the api and use the statsd in the utils library. The statsd code was added to the utils library a while ago, uses the statsd from the util library and therefore consolidates the code into once place. --- app/celery/letters_pdf_tasks.py | 2 +- app/celery/process_ses_receipts_tasks.py | 3 +- app/celery/provider_tasks.py | 2 +- app/celery/scheduled_tasks.py | 2 +- app/celery/service_callback_tasks.py | 4 +- app/celery/statistics_tasks.py | 2 +- app/celery/tasks.py | 2 +- app/dao/inbound_sms_dao.py | 2 +- app/dao/jobs_dao.py | 2 +- app/dao/monthly_billing_dao.py | 2 +- app/dao/notification_usage_dao.py | 2 +- app/dao/notifications_dao.py | 2 +- app/dao/services_dao.py | 2 +- app/dao/statistics_dao.py | 2 +- app/dao/stats_template_usage_by_month_dao.py | 2 +- app/statsd_decorators.py | 40 -------------------- tests/app/test_statsd_decorators.py | 24 ------------ 17 files changed, 18 insertions(+), 79 deletions(-) delete mode 100644 app/statsd_decorators.py delete mode 100644 tests/app/test_statsd_decorators.py diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 0404a3a4c..69b4a4add 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta import math from flask import current_app +from notifications_utils.statsd_decorators import statsd from requests import ( post as requests_post, RequestException @@ -20,7 +21,6 @@ from app.dao.notifications_dao import ( dao_get_notifications_by_references, ) from app.models import NOTIFICATION_CREATED -from app.statsd_decorators import statsd LETTERS_PDF_FILE_LOCATION_STRUCTURE = \ '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf' diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 6f59a33f5..d0279eb77 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -1,8 +1,9 @@ from flask import current_app +from notifications_utils.statsd_decorators import statsd from app import notify_celery from app.config import QueueNames -from app.statsd_decorators import statsd + from app.notifications.notifications_ses_callback import process_ses_response diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index 5f1f81be1..50cfd8506 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -1,13 +1,13 @@ from celery.signals import worker_process_shutdown from flask import current_app from notifications_utils.recipients import InvalidEmailError +from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound from app import notify_celery from app.config import QueueNames from app.dao import notifications_dao from app.dao.notifications_dao import update_notification_status_by_id -from app.statsd_decorators import statsd from app.delivery import send_to_providers diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 579927b83..9a9ed68a8 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -6,6 +6,7 @@ from datetime import ( from celery.signals import worker_process_shutdown from flask import current_app +from notifications_utils.statsd_decorators import statsd from sqlalchemy import and_ from sqlalchemy.exc import SQLAlchemyError from notifications_utils.s3 import s3upload @@ -54,7 +55,6 @@ from app.models import ( JOB_STATUS_READY_TO_SEND ) from app.notifications.process_notifications import send_notification_to_queue -from app.statsd_decorators import statsd from app.celery.tasks import ( create_dvla_file_contents_for_notifications, process_job diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index 19f322842..52d86f631 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -1,4 +1,7 @@ import json + +from notifications_utils.statsd_decorators import statsd + from app import ( DATETIME_FORMAT, notify_celery, @@ -7,7 +10,6 @@ from app.dao.notifications_dao import ( get_notification_by_id, ) -from app.statsd_decorators import statsd from app.dao.service_callback_api_dao import get_service_callback_api_for_service from requests import ( HTTPError, diff --git a/app/celery/statistics_tasks.py b/app/celery/statistics_tasks.py index dd80d32fa..141f829b0 100644 --- a/app/celery/statistics_tasks.py +++ b/app/celery/statistics_tasks.py @@ -1,10 +1,10 @@ from celery.signals import worker_process_shutdown +from notifications_utils.statsd_decorators import statsd from sqlalchemy.exc import SQLAlchemyError from app import notify_celery from flask import current_app -from app.statsd_decorators import statsd from app.dao.statistics_dao import ( create_or_update_job_sending_statistics, update_job_stats_outcome_count diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 7063b791a..76d7c7e7a 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -8,6 +8,7 @@ from flask import current_app from notifications_utils.recipients import ( RecipientCSV ) +from notifications_utils.statsd_decorators import statsd from notifications_utils.template import ( SMSMessageTemplate, WithSubjectTemplate, @@ -71,7 +72,6 @@ from app.models import ( ) from app.notifications.process_notifications import persist_notification from app.service.utils import service_allowed_to_send_to -from app.statsd_decorators import statsd from notifications_utils.s3 import s3upload diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py index f7d54281e..4bd2a417d 100644 --- a/app/dao/inbound_sms_dao.py +++ b/app/dao/inbound_sms_dao.py @@ -3,12 +3,12 @@ from datetime import ( datetime ) from flask import current_app +from notifications_utils.statsd_decorators import statsd from sqlalchemy import desc from app import db from app.dao.dao_utils import transactional from app.models import InboundSms -from app.statsd_decorators import statsd @transactional diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index 4b2e417b1..6eb2f449e 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -2,6 +2,7 @@ import uuid from datetime import datetime, timedelta from flask import current_app +from notifications_utils.statsd_decorators import statsd from sqlalchemy import func, desc, asc, cast, Date as sql_date from app import db @@ -12,7 +13,6 @@ from app.models import ( LETTER_TYPE ) from app.variables import LETTER_TEST_API_FILENAME -from app.statsd_decorators import statsd @statsd(namespace="dao") diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index aeda659b6..dafbbb104 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -1,5 +1,6 @@ from datetime import datetime +from notifications_utils.statsd_decorators import statsd from app import db from app.dao.dao_utils import transactional @@ -12,7 +13,6 @@ from app.models import ( MonthlyBilling, NotificationHistory ) -from app.statsd_decorators import statsd from app.utils import convert_utc_to_bst diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 1bb327af8..7105f5b3e 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta +from notifications_utils.statsd_decorators import statsd from sqlalchemy import Float, Integer, and_ from sqlalchemy import func, case, cast from sqlalchemy import literal_column @@ -17,7 +18,6 @@ from app.models import ( LetterRate, Service ) -from app.statsd_decorators import statsd from app.utils import get_london_month_from_utc_column diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 886748847..349cd704b 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -13,6 +13,7 @@ from notifications_utils.recipients import ( InvalidPhoneError, InvalidEmailError, ) +from notifications_utils.statsd_decorators import statsd from werkzeug.datastructures import MultiDict from sqlalchemy import (desc, func, or_, asc) from sqlalchemy.orm import joinedload @@ -44,7 +45,6 @@ from app.models import ( ) from app.dao.dao_utils import transactional -from app.statsd_decorators import statsd @statsd(namespace="dao") diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 95f83dec2..046f10007 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,6 +1,7 @@ import uuid from datetime import date, datetime, timedelta, time +from notifications_utils.statsd_decorators import statsd from sqlalchemy import asc, func, extract from sqlalchemy.orm import joinedload from flask import current_app @@ -40,7 +41,6 @@ from app.models import ( TEMPLATE_TYPES, LETTER_TYPE, ) -from app.statsd_decorators import statsd from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc DEFAULT_SERVICE_PERMISSIONS = [ diff --git a/app/dao/statistics_dao.py b/app/dao/statistics_dao.py index d1e1cc661..94f2edc85 100644 --- a/app/dao/statistics_dao.py +++ b/app/dao/statistics_dao.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta from itertools import groupby from flask import current_app +from notifications_utils.statsd_decorators import statsd from sqlalchemy import func from sqlalchemy.exc import IntegrityError, SQLAlchemyError @@ -17,7 +18,6 @@ from app.models import ( NOTIFICATION_STATUS_SUCCESS, NOTIFICATION_DELIVERED, NOTIFICATION_SENT) -from app.statsd_decorators import statsd @transactional diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py index aa88924f9..cc54e16c1 100644 --- a/app/dao/stats_template_usage_by_month_dao.py +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -1,7 +1,7 @@ +from notifications_utils.statsd_decorators import statsd from sqlalchemy import or_, and_ from app import db -from app.statsd_decorators import statsd from app.dao.dao_utils import transactional from app.models import StatsTemplateUsageByMonth, Template diff --git a/app/statsd_decorators.py b/app/statsd_decorators.py deleted file mode 100644 index bb21a0562..000000000 --- a/app/statsd_decorators.py +++ /dev/null @@ -1,40 +0,0 @@ -import functools - -from app import statsd_client -from flask import current_app -from monotonic import monotonic - - -def statsd(namespace): - def time_function(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - start_time = monotonic() - try: - res = func(*args, **kwargs) - elapsed_time = monotonic() - start_time - statsd_client.incr('{namespace}.{func}'.format( - namespace=namespace, func=func.__name__) - ) - statsd_client.timing('{namespace}.{func}'.format( - namespace=namespace, func=func.__name__), elapsed_time - ) - - except Exception as e: - current_app.logger.error( - "{namespace} call {func} failed".format( - namespace=namespace, func=func.__name__ - ) - ) - raise e - else: - current_app.logger.info( - "{namespace} call {func} took {time}".format( - namespace=namespace, func=func.__name__, time="{0:.4f}".format(elapsed_time) - ) - ) - return res - wrapper.__wrapped__.__name__ = func.__name__ - return wrapper - - return time_function diff --git a/tests/app/test_statsd_decorators.py b/tests/app/test_statsd_decorators.py deleted file mode 100644 index dc904f2e1..000000000 --- a/tests/app/test_statsd_decorators.py +++ /dev/null @@ -1,24 +0,0 @@ -from unittest.mock import ANY - -from app.statsd_decorators import statsd -import app - - -class AnyStringWith(str): - def __eq__(self, other): - return self in other - - -def test_should_call_statsd(notify_api, mocker): - mocker.patch('app.statsd_client.incr') - mocker.patch('app.statsd_client.timing') - mock_logger = mocker.patch.object(notify_api.logger, 'info') - - @statsd(namespace="test") - def test_function(): - return True - - assert test_function() - app.statsd_client.incr.assert_called_once_with("test.test_function") - app.statsd_client.timing.assert_called_once_with("test.test_function", ANY) - mock_logger.assert_called_once_with(AnyStringWith("test call test_function took "))