From e3e067c79544b185243a37a9faa9c645ee5dcbb5 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Mon, 12 Apr 2021 15:19:18 +0100 Subject: [PATCH] Remove redundant @statsd timing decorators These are superseded by timing task execution generically in the NotifyTask superclass [1]. Note that we need to wait until we've gathered enough data under the new metrics before removing these. [1]: https://github.com/alphagov/notifications-api/pull/3201#pullrequestreview-633549376 --- app/celery/broadcast_message_tasks.py | 3 --- app/celery/letters_pdf_tasks.py | 8 -------- app/celery/nightly_tasks.py | 12 ------------ app/celery/process_ses_receipts_tasks.py | 2 -- app/celery/process_sms_client_response_tasks.py | 2 -- app/celery/provider_tasks.py | 3 --- app/celery/reporting_tasks.py | 5 ----- app/celery/scheduled_tasks.py | 11 ----------- app/celery/service_callback_tasks.py | 3 --- app/celery/tasks.py | 14 -------------- 10 files changed, 63 deletions(-) diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py index a319027ec..d6cf65842 100644 --- a/app/celery/broadcast_message_tasks.py +++ b/app/celery/broadcast_message_tasks.py @@ -2,7 +2,6 @@ import uuid from datetime import datetime from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy.schema import Sequence from app import cbc_proxy_client, db, notify_celery, zendesk_client @@ -104,7 +103,6 @@ def check_provider_message_should_send(broadcast_event, provider): @notify_celery.task(name="send-broadcast-event") -@statsd(namespace="tasks") def send_broadcast_event(broadcast_event_id): if not current_app.config['CBC_PROXY_ENABLED']: current_app.logger.info(f'CBC Proxy disabled, not sending broadcast_event {broadcast_event_id}') @@ -147,7 +145,6 @@ def send_broadcast_event(broadcast_event_id): # max_retries=None: retry forever @notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None) -@statsd(namespace="tasks") def send_broadcast_provider_message(self, broadcast_event_id, provider): broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id) diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 9307bc8ec..951da17cd 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -6,7 +6,6 @@ from botocore.exceptions import ClientError as BotoClientError from flask import current_app from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE from notifications_utils.postal_address import PostalAddress -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_utc_to_bst from app import encryption, notify_celery @@ -56,7 +55,6 @@ from app.models import ( @notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300) -@statsd(namespace="tasks") def get_pdf_for_templated_letter(self, notification_id): try: notification = get_notification_by_id(notification_id, _raise=True) @@ -102,7 +100,6 @@ def get_pdf_for_templated_letter(self, notification_id): @notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300) -@statsd(namespace="tasks") def update_billable_units_for_letter(self, notification_id, page_count): notification = get_notification_by_id(notification_id, _raise=True) @@ -120,7 +117,6 @@ def update_billable_units_for_letter(self, notification_id, page_count): @notify_celery.task(name='collate-letter-pdfs-to-be-sent') @cronitor("collate-letter-pdfs-to-be-sent") -@statsd(namespace="tasks") def collate_letter_pdfs_to_be_sent(): """ Finds all letters which are still waiting to be sent to DVLA for printing @@ -280,7 +276,6 @@ def group_letters(letter_pdfs): @notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300) -@statsd(namespace="tasks") def sanitise_letter(self, filename): try: reference = get_reference_from_filename(filename) @@ -319,7 +314,6 @@ def sanitise_letter(self, filename): @notify_celery.task(bind=True, name='process-sanitised-letter', max_retries=15, default_retry_delay=300) -@statsd(namespace="tasks") def process_sanitised_letter(self, sanitise_data): letter_details = encryption.decrypt(sanitise_data) @@ -433,7 +427,6 @@ def _move_invalid_letter_and_update_status( @notify_celery.task(name='process-virus-scan-failed') -@statsd(namespace="tasks") def process_virus_scan_failed(filename): move_failed_pdf(filename, ScanErrorType.FAILURE) reference = get_reference_from_filename(filename) @@ -453,7 +446,6 @@ def process_virus_scan_failed(filename): @notify_celery.task(name='process-virus-scan-error') -@statsd(namespace="tasks") def process_virus_scan_error(filename): move_failed_pdf(filename, ScanErrorType.ERROR) reference = get_reference_from_filename(filename) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index 54f0ca059..03b84f7e7 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta import pytz from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy import func from sqlalchemy.exc import SQLAlchemyError @@ -43,14 +42,12 @@ from app.utils import get_london_midnight_in_utc @notify_celery.task(name="remove_sms_email_jobs") @cronitor("remove_sms_email_jobs") -@statsd(namespace="tasks") def remove_sms_email_csv_files(): _remove_csv_files([EMAIL_TYPE, SMS_TYPE]) @notify_celery.task(name="remove_letter_jobs") @cronitor("remove_letter_jobs") -@statsd(namespace="tasks") def remove_letter_csv_files(): _remove_csv_files([LETTER_TYPE]) @@ -64,7 +61,6 @@ def _remove_csv_files(job_types): @notify_celery.task(name="delete-notifications-older-than-retention") -@statsd(namespace="tasks") def delete_notifications_older_than_retention(): delete_email_notifications_older_than_retention() delete_sms_notifications_older_than_retention() @@ -73,7 +69,6 @@ def delete_notifications_older_than_retention(): @notify_celery.task(name="delete-sms-notifications") @cronitor("delete-sms-notifications") -@statsd(namespace="tasks") def delete_sms_notifications_older_than_retention(): try: start = datetime.utcnow() @@ -93,7 +88,6 @@ def delete_sms_notifications_older_than_retention(): @notify_celery.task(name="delete-email-notifications") @cronitor("delete-email-notifications") -@statsd(namespace="tasks") def delete_email_notifications_older_than_retention(): try: start = datetime.utcnow() @@ -113,7 +107,6 @@ def delete_email_notifications_older_than_retention(): @notify_celery.task(name="delete-letter-notifications") @cronitor("delete-letter-notifications") -@statsd(namespace="tasks") def delete_letter_notifications_older_than_retention(): try: start = datetime.utcnow() @@ -133,7 +126,6 @@ def delete_letter_notifications_older_than_retention(): @notify_celery.task(name='timeout-sending-notifications') @cronitor('timeout-sending-notifications') -@statsd(namespace="tasks") def timeout_notifications(): technical_failure_notifications, temporary_failure_notifications = \ dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')) @@ -158,7 +150,6 @@ def timeout_notifications(): @notify_celery.task(name="delete-inbound-sms") @cronitor("delete-inbound-sms") -@statsd(namespace="tasks") def delete_inbound_sms(): try: start = datetime.utcnow() @@ -177,7 +168,6 @@ def delete_inbound_sms(): @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") @cronitor("raise-alert-if-letter-notifications-still-sending") -@statsd(namespace="tasks") def raise_alert_if_letter_notifications_still_sending(): still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() @@ -224,7 +214,6 @@ def get_letter_notifications_still_sending_when_they_shouldnt_be(): @notify_celery.task(name='raise-alert-if-no-letter-ack-file') @cronitor('raise-alert-if-no-letter-ack-file') -@statsd(namespace="tasks") def letter_raise_alert_if_no_ack_file_for_zip(): # get a list of zip files since yesterday zip_file_set = set() @@ -276,7 +265,6 @@ def letter_raise_alert_if_no_ack_file_for_zip(): @notify_celery.task(name='save-daily-notification-processing-time') @cronitor("save-daily-notification-processing-time") -@statsd(namespace="tasks") def save_daily_notification_processing_time(bst_date=None): # bst_date is a string in the format of "YYYY-MM-DD" if bst_date is None: diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index f74581304..21968740a 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta import iso8601 from celery.exceptions import Retry from flask import current_app, json -from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound from app import notify_celery, statsd_client @@ -20,7 +19,6 @@ from app.notifications.notifications_ses_callback import ( @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def process_ses_results(self, response): try: ses_message = json.loads(response['Message']) diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index 1aef5c699..5c7c67ecb 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -2,7 +2,6 @@ import uuid from datetime import datetime from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.template import SMSMessageTemplate from app import notify_celery, statsd_client @@ -28,7 +27,6 @@ sms_response_mapper = { @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None): # validate reference try: diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index 841b9b63e..1b1bf6894 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -1,5 +1,4 @@ from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound from app import notify_celery @@ -15,7 +14,6 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300) -@statsd(namespace="tasks") def deliver_sms(self, notification_id): try: current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id)) @@ -46,7 +44,6 @@ def deliver_sms(self, notification_id): @notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300) -@statsd(namespace="tasks") def deliver_email(self, notification_id): try: current_app.logger.info("Start sending email for notification id: {}".format(notification_id)) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 38f25f805..1645f8b25 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from flask import current_app -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_utc_to_bst from app import notify_celery @@ -20,7 +19,6 @@ from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE @notify_celery.task(name="create-nightly-billing") @cronitor("create-nightly-billing") -@statsd(namespace="tasks") def create_nightly_billing(day_start=None): current_app.logger.info("create-nightly-billing task: started") # day_start is a datetime.date() object. e.g. @@ -43,7 +41,6 @@ def create_nightly_billing(day_start=None): @notify_celery.task(name="create-nightly-billing-for-day") -@statsd(namespace="tasks") def create_nightly_billing_for_day(process_day): process_day = datetime.strptime(process_day, "%Y-%m-%d").date() current_app.logger.info( @@ -69,7 +66,6 @@ def create_nightly_billing_for_day(process_day): @notify_celery.task(name="create-nightly-notification-status") @cronitor("create-nightly-notification-status") -@statsd(namespace="tasks") def create_nightly_notification_status(): current_app.logger.info("create-nightly-notification-status task: started") yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1) @@ -100,7 +96,6 @@ def create_nightly_notification_status(): @notify_celery.task(name="create-nightly-notification-status-for-day") -@statsd(namespace="tasks") def create_nightly_notification_status_for_day(process_day, notification_type): process_day = datetime.strptime(process_day, "%Y-%m-%d").date() current_app.logger.info( diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 3c8f939d4..b5fece6b0 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from flask import current_app -from notifications_utils.statsd_decorators import statsd from sqlalchemy import between from sqlalchemy.exc import SQLAlchemyError @@ -55,7 +54,6 @@ from app.notifications.process_notifications import send_notification_to_queue @notify_celery.task(name="run-scheduled-jobs") -@statsd(namespace="tasks") def run_scheduled_jobs(): try: for job in dao_set_scheduled_jobs_to_pending(): @@ -67,7 +65,6 @@ def run_scheduled_jobs(): @notify_celery.task(name="delete-verify-codes") -@statsd(namespace="tasks") def delete_verify_codes(): try: start = datetime.utcnow() @@ -81,7 +78,6 @@ def delete_verify_codes(): @notify_celery.task(name="delete-invitations") -@statsd(namespace="tasks") def delete_invitations(): try: start = datetime.utcnow() @@ -96,7 +92,6 @@ def delete_invitations(): @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery') -@statsd(namespace="tasks") def switch_current_sms_provider_on_slow_delivery(): """ Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered @@ -119,13 +114,11 @@ def switch_current_sms_provider_on_slow_delivery(): @notify_celery.task(name='tend-providers-back-to-middle') -@statsd(namespace='tasks') def tend_providers_back_to_middle(): dao_adjust_provider_priority_back_to_resting_points() @notify_celery.task(name='check-job-status') -@statsd(namespace="tasks") def check_job_status(): """ every x minutes do this check @@ -175,7 +168,6 @@ def check_job_status(): @notify_celery.task(name='replay-created-notifications') -@statsd(namespace="tasks") def replay_created_notifications(): # if the notification has not be send after 1 hour, then try to resend. resend_created_notifications_older_than = (60 * 60) @@ -209,7 +201,6 @@ def replay_created_notifications(): @notify_celery.task(name='check-if-letters-still-pending-virus-check') -@statsd(namespace="tasks") def check_if_letters_still_pending_virus_check(): letters = dao_precompiled_letters_still_pending_virus_check() @@ -231,7 +222,6 @@ def check_if_letters_still_pending_virus_check(): @notify_celery.task(name='check-if-letters-still-in-created') -@statsd(namespace="tasks") def check_if_letters_still_in_created(): letters = dao_old_letters_with_created_status() @@ -268,7 +258,6 @@ def check_for_missing_rows_in_completed_jobs(): @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers') -@statsd(namespace="tasks") def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(): start_date = (datetime.utcnow() - timedelta(days=1)) end_date = datetime.utcnow() diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index 6cc17f0df..a62247b8e 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -1,7 +1,6 @@ import json from flask import current_app -from notifications_utils.statsd_decorators import statsd from requests import HTTPError, RequestException, request from app import encryption, notify_celery @@ -10,7 +9,6 @@ from app.utils import DATETIME_FORMAT @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def send_delivery_status_to_service( self, notification_id, encrypted_status_update ): @@ -39,7 +37,6 @@ def send_delivery_status_to_service( @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def send_complaint_to_service(self, complaint_data): complaint = encryption.decrypt(complaint_data) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 91997b2fd..a69b77b20 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -6,7 +6,6 @@ from flask import current_app from notifications_utils.columns import Columns from notifications_utils.postal_address import PostalAddress from notifications_utils.recipients import RecipientCSV -from notifications_utils.statsd_decorators import statsd from notifications_utils.timezones import convert_utc_to_bst from requests import HTTPError, RequestException, request from sqlalchemy.exc import IntegrityError, SQLAlchemyError @@ -62,7 +61,6 @@ from app.utils import DATETIME_FORMAT, get_reference_from_personalisation @notify_celery.task(name="process-job") -@statsd(namespace="tasks") def process_job(job_id, sender_id=None): start = datetime.utcnow() job = dao_get_job_by_id(job_id) @@ -176,7 +174,6 @@ def __sending_limits_for_job_exceeded(service, job, job_id): @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def save_sms(self, service_id, notification_id, @@ -235,7 +232,6 @@ def save_sms(self, @notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def save_email(self, service_id, notification_id, @@ -287,14 +283,12 @@ def save_email(self, @notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def save_api_email(self, encrypted_notification): save_api_email_or_sms(self, encrypted_notification) @notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def save_api_sms(self, encrypted_notification): save_api_email_or_sms(self, encrypted_notification) @@ -344,7 +338,6 @@ def save_api_email_or_sms(self, encrypted_notification): @notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def save_letter( self, service_id, @@ -407,7 +400,6 @@ def save_letter( @notify_celery.task(bind=True, name='update-letter-notifications-to-sent') -@statsd(namespace="tasks") def update_letter_notifications_to_sent_to_dvla(self, notification_references): # This task will be called by the FTP app to update notifications as sent to DVLA provider = get_provider_details_by_notification_type(LETTER_TYPE)[0] @@ -426,7 +418,6 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references): @notify_celery.task(bind=True, name='update-letter-notifications-to-error') -@statsd(namespace="tasks") def update_letter_notifications_to_error(self, notification_references): # This task will be called by the FTP app to update notifications as sent to DVLA @@ -462,7 +453,6 @@ def handle_exception(task, notification, notification_id, exc): @notify_celery.task(bind=True, name='update-letter-notifications-statuses') -@statsd(namespace="tasks") def update_letter_notifications_statuses(self, filename): notification_updates = parse_dvla_file(filename) @@ -479,7 +469,6 @@ def update_letter_notifications_statuses(self, filename): @notify_celery.task(bind=True, name="record-daily-sorted-counts") -@statsd(namespace="tasks") def record_daily_sorted_counts(self, filename): sorted_letter_counts = defaultdict(int) notification_updates = parse_dvla_file(filename) @@ -566,7 +555,6 @@ def check_billable_units(notification_update): @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def send_inbound_sms_to_service(self, inbound_sms_id, service_id): inbound_api = get_service_inbound_api_for_service(service_id=service_id) if not inbound_api: @@ -621,7 +609,6 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id): @notify_celery.task(name='process-incomplete-jobs') -@statsd(namespace="tasks") def process_incomplete_jobs(job_ids): jobs = [dao_get_job_by_id(job_id) for job_id in job_ids] @@ -658,7 +645,6 @@ def process_incomplete_job(job_id): @notify_celery.task(name='process-returned-letters-list') -@statsd(namespace="tasks") def process_returned_letters_list(notification_references): updated, updated_history = dao_update_notifications_by_reference( notification_references,