From df393e36c5c50521dc0b93235696f0820d1f1e75 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 25 Mar 2021 12:27:24 +0000 Subject: [PATCH] send a p1 when a broadcast goes out on production it's important to keep tabs on when these things leave our system. Sending a zendesk ticket that triggers a P1 is probably our simplest way of notifying the team when this happens (it's what we do with out of hours emergencies on the admin app too). We don't have any direct pagerduty integrations from the api app, but we already have the zendesk client hooked up. After broadcasts go live, we may want to change this to a P2 (but even then, there's arguments for keeping it P1 to start with I think). Don't cause a P1 if it goes out on staging as that might be MNOs testing. --- app/celery/broadcast_message_tasks.py | 26 +++++++++- .../celery/test_broadcast_message_tasks.py | 48 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py index fe0e66c37..8bf1f2316 100644 --- a/app/celery/broadcast_message_tasks.py +++ b/app/celery/broadcast_message_tasks.py @@ -5,7 +5,7 @@ from flask import current_app from notifications_utils.statsd_decorators import statsd from sqlalchemy.schema import Sequence -from app import cbc_proxy_client, db, notify_celery +from app import cbc_proxy_client, db, notify_celery, zendesk_client from app.clients.cbc_proxy import ( CBCProxyFatalException, CBCProxyRetryableException, @@ -111,6 +111,30 @@ def send_broadcast_event(broadcast_event_id): return broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id) + + if current_app.config['NOTIFY_ENVIRONMENT'] == 'production': + broadcast_message = broadcast_event.broadcast_message + # raise a P1 to alert team that broadcast is going out. + message = '\n'.join([ + 'Broadcast Sent', + '', + f'https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}', # noqa + '', + f'This broacast has been sent on channel {broadcast_message.service.broadcast_channel}.', + f'This broadcast is targeted at areas {broadcast_message.areas.get("areas")}.', + '' + f'This broadcast\'s content starts "{broadcast_message.content[:100]}"' + '', + 'If this alert is not expected refer to the runbook for instructions.', + 'https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs', + ]) + zendesk_client.create_ticket( + subject="Live broadcast sent", + message=message, + ticket_type=zendesk_client.TYPE_INCIDENT, + p1=True, + ) + for provider in broadcast_event.service.get_available_broadcast_providers(): send_broadcast_provider_message.apply_async( kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider}, diff --git a/tests/app/celery/test_broadcast_message_tasks.py b/tests/app/celery/test_broadcast_message_tasks.py index c8942ee6b..a5cc2ffdb 100644 --- a/tests/app/celery/test_broadcast_message_tasks.py +++ b/tests/app/celery/test_broadcast_message_tasks.py @@ -36,6 +36,7 @@ def test_send_broadcast_event_queues_up_for_active_providers(mocker, notify_api, template = create_template(sample_broadcast_service, BROADCAST_TYPE) broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) event = create_broadcast_event(broadcast_message) + mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket") mock_send_broadcast_provider_message = mocker.patch( 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', @@ -49,6 +50,9 @@ def test_send_broadcast_event_queues_up_for_active_providers(mocker, notify_api, call(kwargs={'broadcast_event_id': event.id, 'provider': 'vodafone'}, queue='broadcast-tasks') ] + # we're on test env so this isn't called + assert mock_create_ticket.called is False + def test_send_broadcast_event_only_sends_to_one_provider_if_set_on_service( mocker, @@ -106,6 +110,50 @@ def test_send_broadcast_event_does_nothing_if_cbc_proxy_disabled(mocker, notify_ assert mock_send_broadcast_provider_message.apply_async.called is False +def test_send_broadcast_event_creates_zendesk_p1(mocker, notify_api, sample_broadcast_service): + template = create_template(sample_broadcast_service, BROADCAST_TYPE) + broadcast_message = create_broadcast_message( + template, + status=BroadcastStatusType.BROADCASTING, + areas={'areas': ['wd20-S13002775', 'wd20-S13002773'], 'simple_polygons': []}, + ) + event = create_broadcast_event(broadcast_message) + mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket") + + mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_provider_message') + + with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'production'): + send_broadcast_event(event.id) + + assert mock_create_ticket.call_count == 1 + zendesk_args = mock_create_ticket.call_args[1] + assert zendesk_args['p1'] is True + assert zendesk_args['ticket_type'] == 'incident' + + assert str(broadcast_message.id) in zendesk_args['message'] + assert 'channel severe' in zendesk_args['message'] + assert "areas ['wd20-S13002775', 'wd20-S13002773']" in zendesk_args['message'] + # the start of the content from the broadcast template + assert "Dear Sir/Madam" in zendesk_args['message'] + + +def test_send_broadcast_event_doesnt_create_zendesk_on_staging(mocker, notify_api, sample_broadcast_service): + template = create_template(sample_broadcast_service, BROADCAST_TYPE) + broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) + event = create_broadcast_event(broadcast_message) + mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket") + + mock_send_broadcast_provider_message = mocker.patch( + 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', + ) + + with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'staging'): + send_broadcast_event(event.id) + + assert mock_send_broadcast_provider_message.apply_async.called is True + assert mock_create_ticket.called is False + + @freeze_time('2020-08-01 12:00') @pytest.mark.parametrize('provider,provider_capitalised', [ ['ee', 'EE'],