Remove everything for the performance platform

We no longer will send them any stats so therefore don't need the code
- the code to work out the nightly stats
- the performance platform client
- any configuration for the client
- any nightly tasks that kick off the sending off the stats

We will require a change in cronitor as we no longer will have this task
run meaning we need to delete the cronitor check.

app/__init__.py

@@ -35,9 +35,6 @@ from app.clients.cbc_proxy import CBCProxyClient
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.performance_platform.performance_platform_client import (
-    PerformancePlatformClient,
-)
 from app.clients.sms.firetext import FiretextClient
 from app.clients.sms.mmg import MMGClient
 
@@ -66,7 +63,6 @@ encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
-performance_platform_client = PerformancePlatformClient()
 cbc_proxy_client = CBCProxyClient()
 document_download_client = DocumentDownloadClient()
 metrics = GDSMetrics()
@@ -117,7 +113,6 @@ def create_app(application):
     notify_celery.init_app(application)
     encryption.init_app(application)
     redis_store.init_app(application)
-    performance_platform_client.init_app(application)
     document_download_client.init_app(application)
 
     cbc_proxy_client.init_app(application)

app/celery/nightly_tasks.py

@@ -6,7 +6,7 @@ from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, performance_platform_client, zendesk_client
+from app import notify_celery, zendesk_client
 from app.aws import s3
 from app.celery.service_callback_tasks import (
     create_delivery_status_callback_data,
@@ -21,7 +21,7 @@ from app.dao.jobs_dao import (
     dao_get_jobs_older_than_data_retention,
 )
 from app.dao.notifications_dao import (
-    dao_get_total_notifications_sent_per_day_for_performance_platform,
+    dao_get_notifications_processing_time_stats,
     dao_timeout_notifications,
     delete_notifications_older_than_retention_by_type,
 )
@@ -38,7 +38,6 @@ from app.models import (
     FactProcessingTime,
     Notification,
 )
-from app.performance_platform import processing_time, total_sent_notifications
 from app.utils import get_london_midnight_in_utc
 
 
@@ -157,54 +156,6 @@ def timeout_notifications():
         raise NotificationTechnicalFailureException(message)
 
 
-@notify_celery.task(name='send-daily-performance-platform-stats')
-@cronitor('send-daily-performance-platform-stats')
-@statsd(namespace="tasks")
-def send_daily_performance_platform_stats(date=None):
-    # date is a string in the format of "YYYY-MM-DD"
-    if date is None:
-        date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        date = datetime.strptime(date, "%Y-%m-%d").date()
-
-    if performance_platform_client.active:
-
-        send_total_sent_notifications_to_performance_platform(bst_date=date)
-        processing_time.send_processing_time_to_performance_platform(bst_date=date)
-
-
-def send_total_sent_notifications_to_performance_platform(bst_date):
-    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
-    start_time = get_london_midnight_in_utc(bst_date)
-
-    email_sent_count = count_dict['email']
-    sms_sent_count = count_dict['sms']
-    letter_sent_count = count_dict['letter']
-
-    current_app.logger.info(
-        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
-        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
-    )
-
-    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-        start_time,
-        'sms',
-        sms_sent_count
-    )
-
-    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-        start_time,
-        'email',
-        email_sent_count
-    )
-
-    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-        start_time,
-        'letter',
-        letter_sent_count
-    )
-
-
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
 @statsd(namespace="tasks")
@@ -336,7 +287,7 @@ def save_daily_notification_processing_time(bst_date=None):
 
     start_time = get_london_midnight_in_utc(bst_date)
     end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
-    result = dao_get_total_notifications_sent_per_day_for_performance_platform(start_time, end_time)
+    result = dao_get_notifications_processing_time_stats(start_time, end_time)
     insert_update_processing_time(
         FactProcessingTime(
             bst_date=bst_date,

app/commands.py

@@ -18,9 +18,6 @@ from sqlalchemy.orm.exc import NoResultFound
 from app import db, encryption
 from app.aws import s3
 from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
-from app.celery.nightly_tasks import (
-    send_total_sent_notifications_to_performance_platform,
-)
 from app.celery.reporting_tasks import (
     create_nightly_notification_status_for_day,
 )
@@ -36,7 +33,6 @@ from app.dao.fact_billing_dao import (
     get_service_ids_that_need_billing_populated,
     update_fact_billing,
 )
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.jobs_dao import dao_get_job_by_id
 from app.dao.organisation_dao import (
     dao_add_service_to_organisation,
@@ -70,21 +66,13 @@ from app.models import (
     SMS_TYPE,
     Domain,
     EmailBranding,
-    FactProcessingTime,
     LetterBranding,
     Notification,
     Organisation,
     Service,
     User,
 )
-from app.performance_platform.processing_time import (
-    send_processing_time_for_start_and_end,
-)
-from app.utils import (
-    DATETIME_FORMAT,
-    get_london_midnight_in_utc,
-    get_midnight_for_day_before,
-)
+from app.utils import DATETIME_FORMAT, get_london_midnight_in_utc
 
 
 @click.group(name='command', help='Additional commands')
@@ -243,59 +231,6 @@ def fix_notification_statuses_not_in_sync():
         result = db.session.execute(subq_hist).fetchall()
 
 
-@notify_command()
-@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
-@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
-def backfill_performance_platform_totals(start_date, end_date):
-    """
-    Send historical total messages sent to Performance Platform.
-
-    WARNING: This does not overwrite existing data. You need to delete
-             the existing data or Performance Platform will double-count.
-    """
-
-    delta = end_date - start_date
-
-    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
-
-    for i in range(delta.days + 1):
-
-        process_date = start_date + timedelta(days=i)
-
-        print('Sending total messages sent for {}'.format(
-            process_date.isoformat()
-        ))
-
-        send_total_sent_notifications_to_performance_platform(process_date)
-
-
-@notify_command()
-@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
-@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
-def backfill_processing_time(start_date, end_date):
-    """
-    Send historical processing time to Performance Platform.
-    """
-
-    delta = end_date - start_date
-
-    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
-
-    for i in range(delta.days + 1):
-        # because the tz conversion funcs talk about midnight, and the midnight before last,
-        # we want to pretend we're running this from the next morning, so add one.
-        process_date = start_date + timedelta(days=i + 1)
-
-        process_start_date = get_midnight_for_day_before(process_date)
-        process_end_date = get_london_midnight_in_utc(process_date)
-
-        print('Sending notification processing-time for {} - {}'.format(
-            process_start_date.isoformat(),
-            process_end_date.isoformat()
-        ))
-        send_processing_time_for_start_and_end(process_start_date, process_end_date, process_date)
-
-
 @notify_command(name='populate-annual-billing')
 @click.option('-y', '--year', required=True, type=int,
               help="""The year to populate the annual billing data for, i.e. 2019""")
@@ -951,78 +886,3 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
-
-
-@notify_command(name='load-processing-time-data')
-@click.option('-f', '--file_name', required=True, help='Text file contain json data for processing time')
-def load_processing_time_data(file_name):
-    # This method loads the data from a text file that was downloaded from
-    # https://www.performance.service.gov.uk/data/govuk-notify/processing-time?flatten=true&duration=30&group_by=status&period=day&collect=count%3Asum&format=json  ## noqa
-    # The data is formatted as a json
-    # {"data": [
-    #     {
-    #       "_count": 1.0,
-    #       "_end_at": "2021-01-27T00:00:00+00:00",
-    #       "_start_at": "2021-01-26T00:00:00+00:00",
-    #       "count:sum": 4024207.0,
-    #       "status": "messages-within-10-secs"
-    #     },
-    #     {
-    #       "_count": 1.0,
-    #       "_end_at": "2021-01-27T00:00:00+00:00",
-    #       "_start_at": "2021-01-26T00:00:00+00:00",
-    #       "count:sum": 4243204.0,
-    #       "status": "messages-total"
-    #     },
-    # ]}
-    #
-    # Using the fact_processing_time_dao.insert_update_processing_time means if this method is run more than once
-    # it will not throw an exception.
-
-    file = open(file_name)
-
-    file_contents = ""
-    for line in file:
-        file_contents += line
-    data = json.loads(file_contents)
-    normalised = []
-
-    class ProcesingTimeData:
-        bst_date = datetime(1990, 1, 1).date()
-        messages_total = 0
-        messages_within_10_secs = 0
-
-        def __eq__(self, obj):
-            return isinstance(obj, ProcesingTimeData) and obj.bst_date == self.bst_date
-
-        def set_bst_date(self, value):
-            self.bst_date = value
-
-        def set_m(self, status, value):
-            if status == 'messages-total':
-                self.messages_total = value
-            elif status == 'messages-within-10-secs':
-                self.messages_within_10_secs = value
-
-    for entry in data['data']:
-        bst_date = datetime.strptime(entry['_start_at'][0:10], "%Y-%m-%d").date()
-        status = entry['status']
-        value = entry['count:sum']
-        obj = ProcesingTimeData()
-        obj.set_bst_date(bst_date)
-        if obj in normalised:
-            normalised[normalised.index(obj)].set_m(status, value)
-        else:
-            d = ProcesingTimeData()
-            d.set_bst_date(bst_date)
-            d.set_m(status, value)
-            normalised.append(d)
-    for n in normalised:
-        print(n.bst_date, n.messages_total, n.messages_within_10_secs)
-        fact_processing_time = FactProcessingTime(bst_date=n.bst_date,
-                                                  messages_total=n.messages_total,
-                                                  messages_within_10_secs=n.messages_within_10_secs
-                                                  )
-        insert_update_processing_time(fact_processing_time)
-
-    print("Done loading processing time data.")

app/config.py

@@ -109,10 +109,6 @@ class Config(object):
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
-    # Performance platform
-    PERFORMANCE_PLATFORM_ENABLED = False
-    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
-
     # Zendesk
     ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')
 
@@ -273,11 +269,6 @@ class Config(object):
             'schedule': crontab(hour=1, minute=40),
             'options': {'queue': QueueNames.PERIODIC}
         },
-        'send-daily-performance-platform-stats': {
-            'task': 'send-daily-performance-platform-stats',
-            'schedule': crontab(hour=2, minute=0),
-            'options': {'queue': QueueNames.PERIODIC}
-        },
         'save-daily-notification-processing-time': {
             'task': 'save-daily-notification-processing-time',
             'schedule': crontab(hour=2, minute=0),
@@ -366,10 +357,6 @@ class Config(object):
 
     HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
 
-    # Format is as follows:
-    # {"dataset_1": "token_1", ...}
-    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
-
     TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
     TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
 
@@ -537,7 +524,6 @@ class Live(Config):
     TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
     FROM_NUMBER = 'GOVUK'
-    PERFORMANCE_PLATFORM_ENABLED = True
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
     SES_STUB_URL = None

app/dao/fact_notification_status_dao.py

@@ -430,18 +430,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     return query.all()
 
 
-def get_total_sent_notifications_for_day_and_type(day, notification_type):
-    result = db.session.query(
-        func.sum(FactNotificationStatus.notification_count).label('count')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.key_type != KEY_TYPE_TEST,
-        FactNotificationStatus.bst_date == day,
-    ).scalar()
-
-    return result or 0
-
-
 def get_total_notifications_for_date_range(start_date, end_date):
     query = db.session.query(
         FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),

app/dao/notifications_dao.py

@@ -684,8 +684,11 @@ def dao_get_notifications_by_references(references):
     ).all()
 
 
-def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+def dao_get_notifications_processing_time_stats(start_date, end_date):
     """
+    For a given time range, returns the number of notifications sent and the number of
+    those notifications that we processed within 10 seconds
+
     SELECT
     count(notifications),
     coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)