send a p1 when a broadcast goes out on production

it's important to keep tabs on when these things leave our system.
Sending a zendesk ticket that triggers a P1 is probably our simplest way
of notifying the team when this happens (it's what we do with out of
hours emergencies on the admin app too). We don't have any direct
pagerduty integrations from the api app, but we already have the zendesk
client hooked up.

After broadcasts go live, we may want to change this to a P2 (but even
then, there's arguments for keeping it P1 to start with I think).

Don't cause a P1 if it goes out on staging as that might be MNOs testing.

app/celery/broadcast_message_tasks.py

@@ -5,7 +5,7 @@ from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.schema import Sequence
 
-from app import cbc_proxy_client, db, notify_celery
+from app import cbc_proxy_client, db, notify_celery, zendesk_client
 from app.clients.cbc_proxy import (
     CBCProxyFatalException,
     CBCProxyRetryableException,
@@ -111,6 +111,30 @@ def send_broadcast_event(broadcast_event_id):
         return
 
     broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
+
+    if current_app.config['NOTIFY_ENVIRONMENT'] == 'production':
+        broadcast_message = broadcast_event.broadcast_message
+        # raise a P1 to alert team that broadcast is going out.
+        message = '\n'.join([
+            'Broadcast Sent',
+            '',
+            f'https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}',  # noqa
+            '',
+            f'This broacast has been sent on channel {broadcast_message.service.broadcast_channel}.',
+            f'This broadcast is targeted at areas {broadcast_message.areas.get("areas")}.',
+            ''
+            f'This broadcast\'s content starts "{broadcast_message.content[:100]}"'
+            '',
+            'If this alert is not expected refer to the runbook for instructions.',
+            'https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs',
+        ])
+        zendesk_client.create_ticket(
+            subject="Live broadcast sent",
+            message=message,
+            ticket_type=zendesk_client.TYPE_INCIDENT,
+            p1=True,
+        )
+
     for provider in broadcast_event.service.get_available_broadcast_providers():
         send_broadcast_provider_message.apply_async(
             kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider},

tests/app/celery/test_broadcast_message_tasks.py

@@ -36,6 +36,7 @@ def test_send_broadcast_event_queues_up_for_active_providers(mocker, notify_api,
     template = create_template(sample_broadcast_service, BROADCAST_TYPE)
     broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING)
     event = create_broadcast_event(broadcast_message)
+    mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket")
 
     mock_send_broadcast_provider_message = mocker.patch(
         'app.celery.broadcast_message_tasks.send_broadcast_provider_message',
@@ -49,6 +50,9 @@ def test_send_broadcast_event_queues_up_for_active_providers(mocker, notify_api,
         call(kwargs={'broadcast_event_id': event.id, 'provider': 'vodafone'}, queue='broadcast-tasks')
     ]
 
+    # we're on test env so this isn't called
+    assert mock_create_ticket.called is False
+
 
 def test_send_broadcast_event_only_sends_to_one_provider_if_set_on_service(
     mocker,
@@ -106,6 +110,50 @@ def test_send_broadcast_event_does_nothing_if_cbc_proxy_disabled(mocker, notify_
     assert mock_send_broadcast_provider_message.apply_async.called is False
 
 
+def test_send_broadcast_event_creates_zendesk_p1(mocker, notify_api, sample_broadcast_service):
+    template = create_template(sample_broadcast_service, BROADCAST_TYPE)
+    broadcast_message = create_broadcast_message(
+        template,
+        status=BroadcastStatusType.BROADCASTING,
+        areas={'areas': ['wd20-S13002775', 'wd20-S13002773'], 'simple_polygons': []},
+    )
+    event = create_broadcast_event(broadcast_message)
+    mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket")
+
+    mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_provider_message')
+
+    with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'production'):
+        send_broadcast_event(event.id)
+
+    assert mock_create_ticket.call_count == 1
+    zendesk_args = mock_create_ticket.call_args[1]
+    assert zendesk_args['p1'] is True
+    assert zendesk_args['ticket_type'] == 'incident'
+
+    assert str(broadcast_message.id) in zendesk_args['message']
+    assert 'channel severe' in zendesk_args['message']
+    assert "areas ['wd20-S13002775', 'wd20-S13002773']" in zendesk_args['message']
+    # the start of the content from the broadcast template
+    assert "Dear Sir/Madam" in zendesk_args['message']
+
+
+def test_send_broadcast_event_doesnt_create_zendesk_on_staging(mocker, notify_api, sample_broadcast_service):
+    template = create_template(sample_broadcast_service, BROADCAST_TYPE)
+    broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING)
+    event = create_broadcast_event(broadcast_message)
+    mock_create_ticket = mocker.patch("app.celery.broadcast_message_tasks.zendesk_client.create_ticket")
+
+    mock_send_broadcast_provider_message = mocker.patch(
+        'app.celery.broadcast_message_tasks.send_broadcast_provider_message',
+    )
+
+    with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'staging'):
+        send_broadcast_event(event.id)
+
+    assert mock_send_broadcast_provider_message.apply_async.called is True
+    assert mock_create_ticket.called is False
+
+
 @freeze_time('2020-08-01 12:00')
 @pytest.mark.parametrize('provider,provider_capitalised', [
     ['ee', 'EE'],