From 9c4205c7c6847acf2dcc182a2a06218449e29126 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 7 Jul 2020 18:02:24 +0100 Subject: [PATCH] Remove statsd decorators from dao functions This done so that we do not use statsd on our http endpoint. We decided we do not need metrics that this gave us. If we change our minds, we will add Prometheus-friendly decorators instead in the future. --- app/dao/fact_billing_dao.py | 17 ---------------- app/dao/fact_notification_status_dao.py | 13 ------------ app/dao/inbound_sms_dao.py | 2 -- app/dao/jobs_dao.py | 2 -- app/dao/notifications_dao.py | 20 ------------------- app/dao/services_dao.py | 4 ---- .../notification_dao/test_notification_dao.py | 14 ------------- tests/app/dao/test_jobs_dao.py | 4 ---- tests/app/dao/test_services_dao.py | 5 ----- 9 files changed, 81 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index e6ee44fa6..ac29ecdbf 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, time, date from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst from sqlalchemy.dialects.postgresql import insert from sqlalchemy import func, desc, Date, Integer, and_ @@ -33,7 +32,6 @@ from app.models import ( from app.utils import get_london_midnight_in_utc, get_notification_table_to_use -@statsd(namespace="dao") def fetch_sms_free_allowance_remainder(start_date): # ASSUMPTION: AnnualBilling has been populated for year. billing_year = get_financial_year_for_datetime(start_date) @@ -64,7 +62,6 @@ def fetch_sms_free_allowance_remainder(start_date): return query -@statsd(namespace="dao") def fetch_sms_billing_for_all_services(start_date, end_date): # ASSUMPTION: AnnualBilling has been populated for year. @@ -117,7 +114,6 @@ def fetch_sms_billing_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_letter_costs_for_all_services(start_date, end_date): query = db.session.query( Organisation.name.label("organisation_name"), @@ -149,7 +145,6 @@ def fetch_letter_costs_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_letter_line_items_for_all_services(start_date, end_date): query = db.session.query( Organisation.name.label("organisation_name"), @@ -185,7 +180,6 @@ def fetch_letter_line_items_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_billing_totals_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) """ @@ -234,7 +228,6 @@ def fetch_billing_totals_for_year(service_id, year): return yearly_data -@statsd(namespace="dao") def fetch_monthly_billing_for_year(service_id, year): year_start_datetime, year_end_datetime = get_financial_year(year) @@ -295,7 +288,6 @@ def fetch_monthly_billing_for_year(service_id, year): return yearly_data -@statsd(namespace="dao") def delete_billing_data_for_service_for_day(process_day, service_id): """ Delete all ft_billing data for a given service on a given bst_date @@ -308,7 +300,6 @@ def delete_billing_data_for_service_for_day(process_day, service_id): ).delete() -@statsd(namespace="dao") def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False): start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) @@ -430,14 +421,12 @@ def _query_for_billing_data(table, notification_type, start_date, end_date, serv return query.all() -@statsd(namespace="dao") def get_rates_for_billing(): non_letter_rates = Rate.query.order_by(desc(Rate.valid_from)).all() letter_rates = LetterRate.query.order_by(desc(LetterRate.start_date)).all() return non_letter_rates, letter_rates -@statsd(namespace="dao") def get_service_ids_that_need_billing_populated(start_date, end_date): return db.session.query( NotificationHistory.service_id @@ -478,7 +467,6 @@ def get_rate( return 0 -@statsd(namespace="dao") def update_fact_billing(data, process_day): non_letter_rates, letter_rates = get_rates_for_billing() rate = get_rate(non_letter_rates, @@ -522,7 +510,6 @@ def update_fact_billing(data, process_day): db.session.commit() -@statsd(namespace="dao") def create_billing_record(data, rate, process_day): billing_record = FactBilling( bst_date=process_day, @@ -540,7 +527,6 @@ def create_billing_record(data, rate, process_day): return billing_record -@statsd(namespace="dao") def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date): query = db.session.query( Service.name.label("service_name"), @@ -566,7 +552,6 @@ def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_email_usage_for_organisation(organisation_id, start_date, end_date): query = db.session.query( Service.name.label("service_name"), @@ -591,7 +576,6 @@ def fetch_email_usage_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date): # ASSUMPTION: AnnualBilling has been populated for year. free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery() @@ -638,7 +622,6 @@ def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_usage_year_for_organisation(organisation_id, year): year_start_datetime, year_end_datetime = get_financial_year(year) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index c2572d70c..ad5fc03c3 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, time from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc from sqlalchemy import case, func, Date from sqlalchemy.dialects.postgresql import insert @@ -35,7 +34,6 @@ from app.utils import ( ) -@statsd(namespace="dao") def fetch_notification_status_for_day(process_day, notification_type): start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) @@ -61,7 +59,6 @@ def fetch_notification_status_for_day(process_day, notification_type): return all_data_for_process_day -@statsd(namespace="dao") def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id): query = db.session.query( table.template_id, @@ -86,7 +83,6 @@ def query_for_fact_status_data(table, start_date, end_date, notification_type, s return query.all() -@statsd(namespace="dao") @transactional def update_fact_notification_status(data, process_day, notification_type): table = FactNotificationStatus.__table__ @@ -109,7 +105,6 @@ def update_fact_notification_status(data, process_day, notification_type): db.session.connection().execute(stmt) -@statsd(namespace="dao") def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return db.session.query( func.date_trunc('month', FactNotificationStatus.bst_date).label('month'), @@ -128,7 +123,6 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service ).all() -@statsd(namespace="dao") def fetch_notification_status_for_service_for_day(bst_day, service_id): return db.session.query( # return current month as a datetime so the data has the same shape as the ft_notification_status query @@ -147,7 +141,6 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id): ).all() -@statsd(namespace="dao") def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7): start_date = midnight_n_days_ago(limit_days) now = datetime.utcnow() @@ -200,7 +193,6 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ ).all() -@statsd(namespace="dao") def fetch_notification_status_totals_for_all_services(start_date, end_date): stats = db.session.query( FactNotificationStatus.notification_type.label('notification_type'), @@ -249,7 +241,6 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_notification_statuses_for_job(job_id): return db.session.query( FactNotificationStatus.notification_status.label('status'), @@ -261,7 +252,6 @@ def fetch_notification_statuses_for_job(job_id): ).all() -@statsd(namespace="dao") def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True): stats = db.session.query( FactNotificationStatus.service_id.label('service_id'), @@ -356,7 +346,6 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro return query.all() -@statsd(namespace="dao") def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service stats = db.session.query( @@ -441,7 +430,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): return query.all() -@statsd(namespace="dao") def get_total_sent_notifications_for_day_and_type(day, notification_type): result = db.session.query( func.sum(FactNotificationStatus.notification_count).label('count') @@ -454,7 +442,6 @@ def get_total_sent_notifications_for_day_and_type(day, notification_type): return result or 0 -@statsd(namespace="dao") def fetch_monthly_notification_statuses_per_service(start_date, end_date): return db.session.query( func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'), diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py index ba17ddd26..04c7c6dae 100644 --- a/app/dao/inbound_sms_dao.py +++ b/app/dao/inbound_sms_dao.py @@ -1,5 +1,4 @@ from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy import desc, and_ from sqlalchemy.orm import aliased from sqlalchemy.dialects.postgresql import insert @@ -108,7 +107,6 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter): return deleted -@statsd(namespace="dao") @transactional def delete_inbound_sms_older_than_retention(): current_app.logger.info('Deleting inbound sms for services with flexible data retention') diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index 745e909ae..ce09fef7b 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from flask import current_app from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES -from notifications_utils.statsd_decorators import statsd from sqlalchemy import ( asc, desc, @@ -32,7 +31,6 @@ from app.models import ( ) -@statsd(namespace="dao") def dao_get_notification_outcomes_for_job(service_id, job_id): notification_statuses = db.session.query( func.count(Notification.status).label('count'), Notification.status diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 8bc62657e..8d43fcf3e 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -14,7 +14,6 @@ from notifications_utils.recipients import ( InvalidEmailError, try_validate_and_format_phone_number ) -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst from sqlalchemy import (desc, func, asc, and_, or_) from sqlalchemy.orm import joinedload @@ -54,7 +53,6 @@ from app.utils import midnight_n_days_ago, escape_special_characters from app.clients.sms.firetext import get_message_status_and_reason_from_firetext_code -@statsd(namespace="dao") def dao_get_last_date_template_was_used(template_id, service_id): last_date_from_notifications = db.session.query( functions.max(Notification.created_at) @@ -121,7 +119,6 @@ def _update_notification_status(notification, status, detailed_status_code=None) return notification -@statsd(namespace="dao") @transactional def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None): notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first() @@ -154,7 +151,6 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta ) -@statsd(namespace="dao") @transactional def update_notification_status_by_reference(reference, status): # this is used to update letters and emails @@ -177,19 +173,16 @@ def update_notification_status_by_reference(reference, status): ) -@statsd(namespace="dao") @transactional def dao_update_notification(notification): notification.updated_at = datetime.utcnow() db.session.add(notification) -@statsd(namespace="dao") def get_notification_for_job(service_id, job_id, notification_id): return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one() -@statsd(namespace="dao") def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None): if page_size is None: page_size = current_app.config['PAGE_SIZE'] @@ -201,12 +194,10 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page ) -@statsd(namespace="dao") def dao_get_notification_count_for_job_id(*, job_id): return Notification.query.filter_by(job_id=job_id).count() -@statsd(namespace="dao") def get_notification_with_personalisation(service_id, notification_id, key_type): filter_dict = {'service_id': service_id, 'id': notification_id} if key_type: @@ -215,7 +206,6 @@ def get_notification_with_personalisation(service_id, notification_id, key_type) return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one() -@statsd(namespace="dao") def get_notification_by_id(notification_id, service_id=None, _raise=False): filters = [Notification.id == notification_id] @@ -231,7 +221,6 @@ def get_notifications(filter_dict=None): return _filter_query(Notification.query, filter_dict=filter_dict) -@statsd(namespace="dao") def get_notifications_for_service( service_id, filter_dict=None, @@ -308,7 +297,6 @@ def _filter_query(query, filter_dict=None): return query -@statsd(namespace="dao") def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=50000): current_app.logger.info( 'Deleting {} notifications for services with flexible data retention'.format(notification_type)) @@ -343,7 +331,6 @@ def delete_notifications_older_than_retention_by_type(notification_type, qry_lim return deleted -@statsd(namespace="dao") @transactional def insert_notification_history_delete_notifications( notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000 @@ -453,7 +440,6 @@ def _delete_letters_from_s3( "Could not delete S3 object with filename: {}".format(s3_object['Key'])) -@statsd(namespace="dao") @transactional def dao_delete_notifications_by_id(notification_id): db.session.query(Notification).filter( @@ -564,7 +550,6 @@ def is_delivery_slow_for_providers( return slow_providers -@statsd(namespace="dao") @transactional def dao_update_notifications_by_reference(references, update_dict): updated_count = Notification.query.filter( @@ -586,7 +571,6 @@ def dao_update_notifications_by_reference(references, update_dict): return updated_count, updated_history_count -@statsd(namespace="dao") def dao_get_notifications_by_recipient_or_reference( service_id, search_term, @@ -648,14 +632,12 @@ def dao_get_notifications_by_recipient_or_reference( return results -@statsd(namespace="dao") def dao_get_notification_by_reference(reference): return Notification.query.filter( Notification.reference == reference ).one() -@statsd(namespace="dao") def dao_get_notification_or_history_by_reference(reference): try: # This try except is necessary because in test keys and research mode does not create notification history. @@ -669,7 +651,6 @@ def dao_get_notification_or_history_by_reference(reference): ).one() -@statsd(namespace="dao") def dao_get_notifications_by_references(references): return Notification.query.filter( Notification.reference.in_(references) @@ -711,7 +692,6 @@ def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date ).one() -@statsd(namespace="dao") def dao_get_last_notification_added_for_job_id(job_id): last_notification_added = Notification.query.filter( Notification.job_id == job_id diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 3aaf6c6b1..a7fb6278e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,7 +1,6 @@ import uuid from datetime import date, datetime, timedelta -from notifications_utils.statsd_decorators import statsd from sqlalchemy.sql.expression import asc, case, and_, func from sqlalchemy.orm import joinedload from sqlalchemy import cast, Float @@ -426,7 +425,6 @@ def delete_service_and_all_associated_db_objects(service): db.session.commit() -@statsd(namespace="dao") def dao_fetch_stats_for_service(service_id, limit_days): # We always want between seven and eight days start_date = midnight_n_days_ago(limit_days) @@ -435,7 +433,6 @@ def dao_fetch_stats_for_service(service_id, limit_days): ).all() -@statsd(namespace="dao") def dao_fetch_todays_stats_for_service(service_id): return _stats_for_service_query(service_id).filter( func.date(Notification.created_at) == date.today() @@ -470,7 +467,6 @@ def _stats_for_service_query(service_id): ) -@statsd(namespace='dao') def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True): today = date.today() start_date = get_london_midnight_in_utc(today) diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index db8cfc7b5..ae52d40d4 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -56,20 +56,6 @@ from tests.app.db import ( ) -def test_should_have_decorated_notifications_dao_functions(): - assert dao_create_notification.__wrapped__.__name__ == 'dao_create_notification' # noqa - assert update_notification_status_by_id.__wrapped__.__name__ == 'update_notification_status_by_id' # noqa - assert dao_update_notification.__wrapped__.__name__ == 'dao_update_notification' # noqa - assert update_notification_status_by_reference.__wrapped__.__name__ == 'update_notification_status_by_reference' # noqa - assert get_notification_for_job.__wrapped__.__name__ == 'get_notification_for_job' # noqa - assert get_notifications_for_job.__wrapped__.__name__ == 'get_notifications_for_job' # noqa - assert get_notification_with_personalisation.__wrapped__.__name__ == 'get_notification_with_personalisation' # noqa - assert get_notifications_for_service.__wrapped__.__name__ == 'get_notifications_for_service' # noqa - assert get_notification_by_id.__wrapped__.__name__ == 'get_notification_by_id' # noqa - assert delete_notifications_older_than_retention_by_type.__wrapped__.__name__ == 'delete_notifications_older_than_retention_by_type' # noqa - assert dao_delete_notifications_by_id.__wrapped__.__name__ == 'dao_delete_notifications_by_id' # noqa - - def test_should_by_able_to_update_status_by_reference(sample_email_template, ses_provider): data = _notification_json(sample_email_template, status='sending') diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index baa6746d4..fccb9b96c 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -28,10 +28,6 @@ from app.models import ( from tests.app.db import create_job, create_service, create_template, create_notification, create_service_contact_list -def test_should_have_decorated_notifications_dao_functions(): - assert dao_get_notification_outcomes_for_job.__wrapped__.__name__ == 'dao_get_notification_outcomes_for_job' # noqa - - def test_should_count_of_statuses_for_notifications_associated_with_job(sample_template, sample_job): create_notification(sample_template, job=sample_job, status='created') create_notification(sample_template, job=sample_job, status='created') diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index a0e080bfe..215f4dc86 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -54,11 +54,6 @@ from tests.app.db import (create_annual_billing, create_api_key, create_template_folder, create_user) -def test_should_have_decorated_services_dao_functions(): - assert dao_fetch_todays_stats_for_service.__wrapped__.__name__ == 'dao_fetch_todays_stats_for_service' # noqa - assert dao_fetch_stats_for_service.__wrapped__.__name__ == 'dao_fetch_stats_for_service' # noqa - - def test_create_service(notify_db_session): user = create_user() create_letter_branding()