Remove redundant @statsd timing decorators

These are superseded by timing task execution generically in the
NotifyTask superclass [1]. Note that we need to wait until we've
gathered enough data under the new metrics before removing these.

[1]: https://github.com/alphagov/notifications-api/pull/3201#pullrequestreview-633549376

app/celery/broadcast_message_tasks.py

@@ -2,7 +2,6 @@ import uuid
 from datetime import datetime
 
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.schema import Sequence
 
 from app import cbc_proxy_client, db, notify_celery, zendesk_client
@@ -104,7 +103,6 @@ def check_provider_message_should_send(broadcast_event, provider):
 
 
 @notify_celery.task(name="send-broadcast-event")
-@statsd(namespace="tasks")
 def send_broadcast_event(broadcast_event_id):
     if not current_app.config['CBC_PROXY_ENABLED']:
         current_app.logger.info(f'CBC Proxy disabled, not sending broadcast_event {broadcast_event_id}')
@@ -147,7 +145,6 @@ def send_broadcast_event(broadcast_event_id):
 
 # max_retries=None: retry forever
 @notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None)
-@statsd(namespace="tasks")
 def send_broadcast_provider_message(self, broadcast_event_id, provider):
     broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
 

app/celery/letters_pdf_tasks.py

@@ -6,7 +6,6 @@ from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
 from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.postal_address import PostalAddress
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import encryption, notify_celery
@@ -56,7 +55,6 @@ from app.models import (
 
 
 @notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300)
-@statsd(namespace="tasks")
 def get_pdf_for_templated_letter(self, notification_id):
     try:
         notification = get_notification_by_id(notification_id, _raise=True)
@@ -102,7 +100,6 @@ def get_pdf_for_templated_letter(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300)
-@statsd(namespace="tasks")
 def update_billable_units_for_letter(self, notification_id, page_count):
     notification = get_notification_by_id(notification_id, _raise=True)
 
@@ -120,7 +117,6 @@ def update_billable_units_for_letter(self, notification_id, page_count):
 
 @notify_celery.task(name='collate-letter-pdfs-to-be-sent')
 @cronitor("collate-letter-pdfs-to-be-sent")
-@statsd(namespace="tasks")
 def collate_letter_pdfs_to_be_sent():
     """
     Finds all letters which are still waiting to be sent to DVLA for printing
@@ -280,7 +276,6 @@ def group_letters(letter_pdfs):
 
 
 @notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300)
-@statsd(namespace="tasks")
 def sanitise_letter(self, filename):
     try:
         reference = get_reference_from_filename(filename)
@@ -319,7 +314,6 @@ def sanitise_letter(self, filename):
 
 
 @notify_celery.task(bind=True, name='process-sanitised-letter', max_retries=15, default_retry_delay=300)
-@statsd(namespace="tasks")
 def process_sanitised_letter(self, sanitise_data):
     letter_details = encryption.decrypt(sanitise_data)
 
@@ -433,7 +427,6 @@ def _move_invalid_letter_and_update_status(
 
 
 @notify_celery.task(name='process-virus-scan-failed')
-@statsd(namespace="tasks")
 def process_virus_scan_failed(filename):
     move_failed_pdf(filename, ScanErrorType.FAILURE)
     reference = get_reference_from_filename(filename)
@@ -453,7 +446,6 @@ def process_virus_scan_failed(filename):
 
 
 @notify_celery.task(name='process-virus-scan-error')
-@statsd(namespace="tasks")
 def process_virus_scan_error(filename):
     move_failed_pdf(filename, ScanErrorType.ERROR)
     reference = get_reference_from_filename(filename)

app/celery/nightly_tasks.py

@@ -2,7 +2,6 @@ from datetime import datetime, timedelta
 
 import pytz
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -43,14 +42,12 @@ from app.utils import get_london_midnight_in_utc
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
-@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
     _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
 
 
 @notify_celery.task(name="remove_letter_jobs")
 @cronitor("remove_letter_jobs")
-@statsd(namespace="tasks")
 def remove_letter_csv_files():
     _remove_csv_files([LETTER_TYPE])
 
@@ -64,7 +61,6 @@ def _remove_csv_files(job_types):
 
 
 @notify_celery.task(name="delete-notifications-older-than-retention")
-@statsd(namespace="tasks")
 def delete_notifications_older_than_retention():
     delete_email_notifications_older_than_retention()
     delete_sms_notifications_older_than_retention()
@@ -73,7 +69,6 @@ def delete_notifications_older_than_retention():
 
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
-@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
@@ -93,7 +88,6 @@ def delete_sms_notifications_older_than_retention():
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
-@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
@@ -113,7 +107,6 @@ def delete_email_notifications_older_than_retention():
 
 @notify_celery.task(name="delete-letter-notifications")
 @cronitor("delete-letter-notifications")
-@statsd(namespace="tasks")
 def delete_letter_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
@@ -133,7 +126,6 @@ def delete_letter_notifications_older_than_retention():
 
 @notify_celery.task(name='timeout-sending-notifications')
 @cronitor('timeout-sending-notifications')
-@statsd(namespace="tasks")
 def timeout_notifications():
     technical_failure_notifications, temporary_failure_notifications = \
         dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
@@ -158,7 +150,6 @@ def timeout_notifications():
 
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
-@statsd(namespace="tasks")
 def delete_inbound_sms():
     try:
         start = datetime.utcnow()
@@ -177,7 +168,6 @@ def delete_inbound_sms():
 
 @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
 @cronitor("raise-alert-if-letter-notifications-still-sending")
-@statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
     still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
 
@@ -224,7 +214,6 @@ def get_letter_notifications_still_sending_when_they_shouldnt_be():
 
 @notify_celery.task(name='raise-alert-if-no-letter-ack-file')
 @cronitor('raise-alert-if-no-letter-ack-file')
-@statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
     # get a list of zip files since yesterday
     zip_file_set = set()
@@ -276,7 +265,6 @@ def letter_raise_alert_if_no_ack_file_for_zip():
 
 @notify_celery.task(name='save-daily-notification-processing-time')
 @cronitor("save-daily-notification-processing-time")
-@statsd(namespace="tasks")
 def save_daily_notification_processing_time(bst_date=None):
     # bst_date is a string in the format of "YYYY-MM-DD"
     if bst_date is None:

app/celery/process_ses_receipts_tasks.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
@@ -20,7 +19,6 @@ from app.notifications.notifications_ses_callback import (
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def process_ses_results(self, response):
     try:
         ses_message = json.loads(response['Message'])

app/celery/process_sms_client_response_tasks.py

@@ -2,7 +2,6 @@ import uuid
 from datetime import datetime
 
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
@@ -28,7 +27,6 @@ sms_response_mapper = {
 
 
 @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
     # validate reference
     try:

app/celery/provider_tasks.py

@@ -1,5 +1,4 @@
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
@@ -15,7 +14,6 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE
 
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
-@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
     try:
         current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -46,7 +44,6 @@ def deliver_sms(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
-@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
     try:
         current_app.logger.info("Start sending email for notification id: {}".format(notification_id))

app/celery/reporting_tasks.py

@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import notify_celery
@@ -20,7 +19,6 @@ from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE
 
 @notify_celery.task(name="create-nightly-billing")
 @cronitor("create-nightly-billing")
-@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
     current_app.logger.info("create-nightly-billing task: started")
     # day_start is a datetime.date() object. e.g.
@@ -43,7 +41,6 @@ def create_nightly_billing(day_start=None):
 
 
 @notify_celery.task(name="create-nightly-billing-for-day")
-@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
     current_app.logger.info(
@@ -69,7 +66,6 @@ def create_nightly_billing_for_day(process_day):
 
 @notify_celery.task(name="create-nightly-notification-status")
 @cronitor("create-nightly-notification-status")
-@statsd(namespace="tasks")
 def create_nightly_notification_status():
     current_app.logger.info("create-nightly-notification-status task: started")
     yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
@@ -100,7 +96,6 @@ def create_nightly_notification_status():
 
 
 @notify_celery.task(name="create-nightly-notification-status-for-day")
-@statsd(namespace="tasks")
 def create_nightly_notification_status_for_day(process_day, notification_type):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
     current_app.logger.info(

app/celery/scheduled_tasks.py

@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import between
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -55,7 +54,6 @@ from app.notifications.process_notifications import send_notification_to_queue
 
 
 @notify_celery.task(name="run-scheduled-jobs")
-@statsd(namespace="tasks")
 def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
@@ -67,7 +65,6 @@ def run_scheduled_jobs():
 
 
 @notify_celery.task(name="delete-verify-codes")
-@statsd(namespace="tasks")
 def delete_verify_codes():
     try:
         start = datetime.utcnow()
@@ -81,7 +78,6 @@ def delete_verify_codes():
 
 
 @notify_celery.task(name="delete-invitations")
-@statsd(namespace="tasks")
 def delete_invitations():
     try:
         start = datetime.utcnow()
@@ -96,7 +92,6 @@ def delete_invitations():
 
 
 @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
-@statsd(namespace="tasks")
 def switch_current_sms_provider_on_slow_delivery():
     """
     Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
@@ -119,13 +114,11 @@ def switch_current_sms_provider_on_slow_delivery():
 
 
 @notify_celery.task(name='tend-providers-back-to-middle')
-@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
     dao_adjust_provider_priority_back_to_resting_points()
 
 
 @notify_celery.task(name='check-job-status')
-@statsd(namespace="tasks")
 def check_job_status():
     """
     every x minutes do this check
@@ -175,7 +168,6 @@ def check_job_status():
 
 
 @notify_celery.task(name='replay-created-notifications')
-@statsd(namespace="tasks")
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
     resend_created_notifications_older_than = (60 * 60)
@@ -209,7 +201,6 @@ def replay_created_notifications():
 
 
 @notify_celery.task(name='check-if-letters-still-pending-virus-check')
-@statsd(namespace="tasks")
 def check_if_letters_still_pending_virus_check():
     letters = dao_precompiled_letters_still_pending_virus_check()
 
@@ -231,7 +222,6 @@ def check_if_letters_still_pending_virus_check():
 
 
 @notify_celery.task(name='check-if-letters-still-in-created')
-@statsd(namespace="tasks")
 def check_if_letters_still_in_created():
     letters = dao_old_letters_with_created_status()
 
@@ -268,7 +258,6 @@ def check_for_missing_rows_in_completed_jobs():
 
 
 @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
-@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
     start_date = (datetime.utcnow() - timedelta(days=1))
     end_date = datetime.utcnow()

app/celery/service_callback_tasks.py

@@ -1,7 +1,6 @@
 import json
 
 from flask import current_app
-from notifications_utils.statsd_decorators import statsd
 from requests import HTTPError, RequestException, request
 
 from app import encryption, notify_celery
@@ -10,7 +9,6 @@ from app.utils import DATETIME_FORMAT
 
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
@@ -39,7 +37,6 @@ def send_delivery_status_to_service(
 
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
     complaint = encryption.decrypt(complaint_data)
 

app/celery/tasks.py

@@ -6,7 +6,6 @@ from flask import current_app
 from notifications_utils.columns import Columns
 from notifications_utils.postal_address import PostalAddress
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
 from requests import HTTPError, RequestException, request
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
@@ -62,7 +61,6 @@ from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
 
 
 @notify_celery.task(name="process-job")
-@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
@@ -176,7 +174,6 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
 
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def save_sms(self,
              service_id,
              notification_id,
@@ -235,7 +232,6 @@ def save_sms(self,
 
 
 @notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def save_email(self,
                service_id,
                notification_id,
@@ -287,14 +283,12 @@ def save_email(self,
 
 
 @notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def save_api_email(self, encrypted_notification):
 
     save_api_email_or_sms(self, encrypted_notification)
 
 
 @notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def save_api_sms(self, encrypted_notification):
     save_api_email_or_sms(self, encrypted_notification)
 
@@ -344,7 +338,6 @@ def save_api_email_or_sms(self, encrypted_notification):
 
 
 @notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def save_letter(
         self,
         service_id,
@@ -407,7 +400,6 @@ def save_letter(
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
-@statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
     provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
@@ -426,7 +418,6 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-error')
-@statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
 
@@ -462,7 +453,6 @@ def handle_exception(task, notification, notification_id, exc):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-statuses')
-@statsd(namespace="tasks")
 def update_letter_notifications_statuses(self, filename):
     notification_updates = parse_dvla_file(filename)
 
@@ -479,7 +469,6 @@ def update_letter_notifications_statuses(self, filename):
 
 
 @notify_celery.task(bind=True, name="record-daily-sorted-counts")
-@statsd(namespace="tasks")
 def record_daily_sorted_counts(self, filename):
     sorted_letter_counts = defaultdict(int)
     notification_updates = parse_dvla_file(filename)
@@ -566,7 +555,6 @@ def check_billable_units(notification_update):
 
 
 @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
-@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
     inbound_api = get_service_inbound_api_for_service(service_id=service_id)
     if not inbound_api:
@@ -621,7 +609,6 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
 
 
 @notify_celery.task(name='process-incomplete-jobs')
-@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
     jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
 
@@ -658,7 +645,6 @@ def process_incomplete_job(job_id):
 
 
 @notify_celery.task(name='process-returned-letters-list')
-@statsd(namespace="tasks")
 def process_returned_letters_list(notification_references):
     updated, updated_history = dao_update_notifications_by_reference(
         notification_references,