diff --git a/app/config.py b/app/config.py index b0e3b555c..81bab624e 100644 --- a/app/config.py +++ b/app/config.py @@ -505,9 +505,6 @@ class Live(Config): CRONITOR_ENABLED = True - # TODO: revert when we're confident statsd won't break things - STATSD_ENABLED = False - class CloudFoundryConfig(Config): pass diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index e6ee44fa6..ac29ecdbf 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, time, date from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst from sqlalchemy.dialects.postgresql import insert from sqlalchemy import func, desc, Date, Integer, and_ @@ -33,7 +32,6 @@ from app.models import ( from app.utils import get_london_midnight_in_utc, get_notification_table_to_use -@statsd(namespace="dao") def fetch_sms_free_allowance_remainder(start_date): # ASSUMPTION: AnnualBilling has been populated for year. billing_year = get_financial_year_for_datetime(start_date) @@ -64,7 +62,6 @@ def fetch_sms_free_allowance_remainder(start_date): return query -@statsd(namespace="dao") def fetch_sms_billing_for_all_services(start_date, end_date): # ASSUMPTION: AnnualBilling has been populated for year. @@ -117,7 +114,6 @@ def fetch_sms_billing_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_letter_costs_for_all_services(start_date, end_date): query = db.session.query( Organisation.name.label("organisation_name"), @@ -149,7 +145,6 @@ def fetch_letter_costs_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_letter_line_items_for_all_services(start_date, end_date): query = db.session.query( Organisation.name.label("organisation_name"), @@ -185,7 +180,6 @@ def fetch_letter_line_items_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_billing_totals_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) """ @@ -234,7 +228,6 @@ def fetch_billing_totals_for_year(service_id, year): return yearly_data -@statsd(namespace="dao") def fetch_monthly_billing_for_year(service_id, year): year_start_datetime, year_end_datetime = get_financial_year(year) @@ -295,7 +288,6 @@ def fetch_monthly_billing_for_year(service_id, year): return yearly_data -@statsd(namespace="dao") def delete_billing_data_for_service_for_day(process_day, service_id): """ Delete all ft_billing data for a given service on a given bst_date @@ -308,7 +300,6 @@ def delete_billing_data_for_service_for_day(process_day, service_id): ).delete() -@statsd(namespace="dao") def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False): start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) @@ -430,14 +421,12 @@ def _query_for_billing_data(table, notification_type, start_date, end_date, serv return query.all() -@statsd(namespace="dao") def get_rates_for_billing(): non_letter_rates = Rate.query.order_by(desc(Rate.valid_from)).all() letter_rates = LetterRate.query.order_by(desc(LetterRate.start_date)).all() return non_letter_rates, letter_rates -@statsd(namespace="dao") def get_service_ids_that_need_billing_populated(start_date, end_date): return db.session.query( NotificationHistory.service_id @@ -478,7 +467,6 @@ def get_rate( return 0 -@statsd(namespace="dao") def update_fact_billing(data, process_day): non_letter_rates, letter_rates = get_rates_for_billing() rate = get_rate(non_letter_rates, @@ -522,7 +510,6 @@ def update_fact_billing(data, process_day): db.session.commit() -@statsd(namespace="dao") def create_billing_record(data, rate, process_day): billing_record = FactBilling( bst_date=process_day, @@ -540,7 +527,6 @@ def create_billing_record(data, rate, process_day): return billing_record -@statsd(namespace="dao") def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date): query = db.session.query( Service.name.label("service_name"), @@ -566,7 +552,6 @@ def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_email_usage_for_organisation(organisation_id, start_date, end_date): query = db.session.query( Service.name.label("service_name"), @@ -591,7 +576,6 @@ def fetch_email_usage_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date): # ASSUMPTION: AnnualBilling has been populated for year. free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery() @@ -638,7 +622,6 @@ def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_usage_year_for_organisation(organisation_id, year): year_start_datetime, year_end_datetime = get_financial_year(year) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index c2572d70c..ad5fc03c3 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, time from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc from sqlalchemy import case, func, Date from sqlalchemy.dialects.postgresql import insert @@ -35,7 +34,6 @@ from app.utils import ( ) -@statsd(namespace="dao") def fetch_notification_status_for_day(process_day, notification_type): start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) @@ -61,7 +59,6 @@ def fetch_notification_status_for_day(process_day, notification_type): return all_data_for_process_day -@statsd(namespace="dao") def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id): query = db.session.query( table.template_id, @@ -86,7 +83,6 @@ def query_for_fact_status_data(table, start_date, end_date, notification_type, s return query.all() -@statsd(namespace="dao") @transactional def update_fact_notification_status(data, process_day, notification_type): table = FactNotificationStatus.__table__ @@ -109,7 +105,6 @@ def update_fact_notification_status(data, process_day, notification_type): db.session.connection().execute(stmt) -@statsd(namespace="dao") def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return db.session.query( func.date_trunc('month', FactNotificationStatus.bst_date).label('month'), @@ -128,7 +123,6 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service ).all() -@statsd(namespace="dao") def fetch_notification_status_for_service_for_day(bst_day, service_id): return db.session.query( # return current month as a datetime so the data has the same shape as the ft_notification_status query @@ -147,7 +141,6 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id): ).all() -@statsd(namespace="dao") def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7): start_date = midnight_n_days_ago(limit_days) now = datetime.utcnow() @@ -200,7 +193,6 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ ).all() -@statsd(namespace="dao") def fetch_notification_status_totals_for_all_services(start_date, end_date): stats = db.session.query( FactNotificationStatus.notification_type.label('notification_type'), @@ -249,7 +241,6 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date): return query.all() -@statsd(namespace="dao") def fetch_notification_statuses_for_job(job_id): return db.session.query( FactNotificationStatus.notification_status.label('status'), @@ -261,7 +252,6 @@ def fetch_notification_statuses_for_job(job_id): ).all() -@statsd(namespace="dao") def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True): stats = db.session.query( FactNotificationStatus.service_id.label('service_id'), @@ -356,7 +346,6 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro return query.all() -@statsd(namespace="dao") def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service stats = db.session.query( @@ -441,7 +430,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): return query.all() -@statsd(namespace="dao") def get_total_sent_notifications_for_day_and_type(day, notification_type): result = db.session.query( func.sum(FactNotificationStatus.notification_count).label('count') @@ -454,7 +442,6 @@ def get_total_sent_notifications_for_day_and_type(day, notification_type): return result or 0 -@statsd(namespace="dao") def fetch_monthly_notification_statuses_per_service(start_date, end_date): return db.session.query( func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'), diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py index ba17ddd26..04c7c6dae 100644 --- a/app/dao/inbound_sms_dao.py +++ b/app/dao/inbound_sms_dao.py @@ -1,5 +1,4 @@ from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy import desc, and_ from sqlalchemy.orm import aliased from sqlalchemy.dialects.postgresql import insert @@ -108,7 +107,6 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter): return deleted -@statsd(namespace="dao") @transactional def delete_inbound_sms_older_than_retention(): current_app.logger.info('Deleting inbound sms for services with flexible data retention') diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index 745e909ae..ce09fef7b 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta from flask import current_app from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES -from notifications_utils.statsd_decorators import statsd from sqlalchemy import ( asc, desc, @@ -32,7 +31,6 @@ from app.models import ( ) -@statsd(namespace="dao") def dao_get_notification_outcomes_for_job(service_id, job_id): notification_statuses = db.session.query( func.count(Notification.status).label('count'), Notification.status diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 6aa976b20..078fa9b6b 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -14,7 +14,6 @@ from notifications_utils.recipients import ( InvalidEmailError, try_validate_and_format_phone_number ) -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst from sqlalchemy import (desc, func, asc, and_, or_) from sqlalchemy.orm import joinedload @@ -54,7 +53,6 @@ from app.utils import midnight_n_days_ago, escape_special_characters from app.clients.sms.firetext import get_message_status_and_reason_from_firetext_code -@statsd(namespace="dao") def dao_get_last_date_template_was_used(template_id, service_id): last_date_from_notifications = db.session.query( functions.max(Notification.created_at) @@ -121,7 +119,6 @@ def _update_notification_status(notification, status, detailed_status_code=None) return notification -@statsd(namespace="dao") @transactional def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None): notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first() @@ -154,7 +151,6 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta ) -@statsd(namespace="dao") @transactional def update_notification_status_by_reference(reference, status): # this is used to update letters and emails @@ -177,19 +173,16 @@ def update_notification_status_by_reference(reference, status): ) -@statsd(namespace="dao") @transactional def dao_update_notification(notification): notification.updated_at = datetime.utcnow() db.session.add(notification) -@statsd(namespace="dao") def get_notification_for_job(service_id, job_id, notification_id): return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one() -@statsd(namespace="dao") def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None): if page_size is None: page_size = current_app.config['PAGE_SIZE'] @@ -201,12 +194,10 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page ) -@statsd(namespace="dao") def dao_get_notification_count_for_job_id(*, job_id): return Notification.query.filter_by(job_id=job_id).count() -@statsd(namespace="dao") def get_notification_with_personalisation(service_id, notification_id, key_type): filter_dict = {'service_id': service_id, 'id': notification_id} if key_type: @@ -215,7 +206,6 @@ def get_notification_with_personalisation(service_id, notification_id, key_type) return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one() -@statsd(namespace="dao") def get_notification_by_id(notification_id, service_id=None, _raise=False): filters = [Notification.id == notification_id] @@ -231,7 +221,6 @@ def get_notifications(filter_dict=None): return _filter_query(Notification.query, filter_dict=filter_dict) -@statsd(namespace="dao") def get_notifications_for_service( service_id, filter_dict=None, @@ -308,7 +297,6 @@ def _filter_query(query, filter_dict=None): return query -@statsd(namespace="dao") def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=50000): current_app.logger.info( 'Deleting {} notifications for services with flexible data retention'.format(notification_type)) @@ -343,7 +331,6 @@ def delete_notifications_older_than_retention_by_type(notification_type, qry_lim return deleted -@statsd(namespace="dao") @transactional def insert_notification_history_delete_notifications( notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000 @@ -453,7 +440,6 @@ def _delete_letters_from_s3( "Could not delete S3 object with filename: {}".format(s3_object['Key'])) -@statsd(namespace="dao") @transactional def dao_delete_notifications_by_id(notification_id): db.session.query(Notification).filter( @@ -564,7 +550,6 @@ def is_delivery_slow_for_providers( return slow_providers -@statsd(namespace="dao") @transactional def dao_update_notifications_by_reference(references, update_dict): updated_count = Notification.query.filter( @@ -586,7 +571,6 @@ def dao_update_notifications_by_reference(references, update_dict): return updated_count, updated_history_count -@statsd(namespace="dao") def dao_get_notifications_by_recipient_or_reference( service_id, search_term, @@ -648,14 +632,12 @@ def dao_get_notifications_by_recipient_or_reference( return results -@statsd(namespace="dao") def dao_get_notification_by_reference(reference): return Notification.query.filter( Notification.reference == reference ).one() -@statsd(namespace="dao") def dao_get_notification_or_history_by_reference(reference): try: # This try except is necessary because in test keys and research mode does not create notification history. @@ -669,7 +651,6 @@ def dao_get_notification_or_history_by_reference(reference): ).one() -@statsd(namespace="dao") def dao_get_notifications_by_references(references): return Notification.query.filter( Notification.reference.in_(references) @@ -711,7 +692,6 @@ def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date ).one() -@statsd(namespace="dao") def dao_get_last_notification_added_for_job_id(job_id): last_notification_added = Notification.query.filter( Notification.job_id == job_id diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 3aaf6c6b1..a7fb6278e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,7 +1,6 @@ import uuid from datetime import date, datetime, timedelta -from notifications_utils.statsd_decorators import statsd from sqlalchemy.sql.expression import asc, case, and_, func from sqlalchemy.orm import joinedload from sqlalchemy import cast, Float @@ -426,7 +425,6 @@ def delete_service_and_all_associated_db_objects(service): db.session.commit() -@statsd(namespace="dao") def dao_fetch_stats_for_service(service_id, limit_days): # We always want between seven and eight days start_date = midnight_n_days_ago(limit_days) @@ -435,7 +433,6 @@ def dao_fetch_stats_for_service(service_id, limit_days): ).all() -@statsd(namespace="dao") def dao_fetch_todays_stats_for_service(service_id): return _stats_for_service_query(service_id).filter( func.date(Notification.created_at) == date.today() @@ -470,7 +467,6 @@ def _stats_for_service_query(service_id): ) -@statsd(namespace='dao') def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True): today = date.today() start_date = get_london_midnight_in_utc(today) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 80c991309..5380dd64c 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -3,6 +3,9 @@ 'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 30, + 'additional_env_vars': { + 'STATSD_HOST': None + }, 'routes': { 'preview': ['api.notify.works'], 'staging': ['api.staging-notify.works'], diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index db8cfc7b5..af51104ca 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -17,7 +17,6 @@ from app.dao.notifications_dao import ( dao_timeout_notifications, dao_update_notification, dao_update_notifications_by_reference, - delete_notifications_older_than_retention_by_type, get_notification_by_id, get_notification_for_job, get_notification_with_personalisation, @@ -56,20 +55,6 @@ from tests.app.db import ( ) -def test_should_have_decorated_notifications_dao_functions(): - assert dao_create_notification.__wrapped__.__name__ == 'dao_create_notification' # noqa - assert update_notification_status_by_id.__wrapped__.__name__ == 'update_notification_status_by_id' # noqa - assert dao_update_notification.__wrapped__.__name__ == 'dao_update_notification' # noqa - assert update_notification_status_by_reference.__wrapped__.__name__ == 'update_notification_status_by_reference' # noqa - assert get_notification_for_job.__wrapped__.__name__ == 'get_notification_for_job' # noqa - assert get_notifications_for_job.__wrapped__.__name__ == 'get_notifications_for_job' # noqa - assert get_notification_with_personalisation.__wrapped__.__name__ == 'get_notification_with_personalisation' # noqa - assert get_notifications_for_service.__wrapped__.__name__ == 'get_notifications_for_service' # noqa - assert get_notification_by_id.__wrapped__.__name__ == 'get_notification_by_id' # noqa - assert delete_notifications_older_than_retention_by_type.__wrapped__.__name__ == 'delete_notifications_older_than_retention_by_type' # noqa - assert dao_delete_notifications_by_id.__wrapped__.__name__ == 'dao_delete_notifications_by_id' # noqa - - def test_should_by_able_to_update_status_by_reference(sample_email_template, ses_provider): data = _notification_json(sample_email_template, status='sending') diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index baa6746d4..fccb9b96c 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -28,10 +28,6 @@ from app.models import ( from tests.app.db import create_job, create_service, create_template, create_notification, create_service_contact_list -def test_should_have_decorated_notifications_dao_functions(): - assert dao_get_notification_outcomes_for_job.__wrapped__.__name__ == 'dao_get_notification_outcomes_for_job' # noqa - - def test_should_count_of_statuses_for_notifications_associated_with_job(sample_template, sample_job): create_notification(sample_template, job=sample_job, status='created') create_notification(sample_template, job=sample_job, status='created') diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index a0e080bfe..215f4dc86 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -54,11 +54,6 @@ from tests.app.db import (create_annual_billing, create_api_key, create_template_folder, create_user) -def test_should_have_decorated_services_dao_functions(): - assert dao_fetch_todays_stats_for_service.__wrapped__.__name__ == 'dao_fetch_todays_stats_for_service' # noqa - assert dao_fetch_stats_for_service.__wrapped__.__name__ == 'dao_fetch_stats_for_service' # noqa - - def test_create_service(notify_db_session): user = create_user() create_letter_branding()