wip

Downgrade exceptions to warnings to reduce emails

app/__init__.py

@@ -60,6 +60,7 @@ performance_platform_client = PerformancePlatformClient()
 cbc_proxy_client = CBCProxyClient()
 document_download_client = DocumentDownloadClient()
 metrics = GDSMetrics()
+counter = 0
 
 notification_provider_clients = NotificationProviderClients()
 
@@ -251,6 +252,7 @@ def register_v2_blueprints(application):
     from app.v2.template.get_template import v2_template_blueprint as get_template
     from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
     from app.v2.template.post_template import v2_template_blueprint as post_template
+    from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint as post_broadcast
     from app.authentication.auth import requires_auth
 
     post_notifications.before_request(requires_auth)
@@ -271,6 +273,9 @@ def register_v2_blueprints(application):
     get_inbound_sms.before_request(requires_auth)
     application.register_blueprint(get_inbound_sms)
 
+    post_broadcast.before_request(requires_auth)
+    application.register_blueprint(post_broadcast)
+
 
 def init_app(app):
 

app/broadcast_message/rest.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 import iso8601
+
 from flask import Blueprint, jsonify, request, current_app
 from app.config import QueueNames
 from app.dao.dao_utils import dao_save_object
@@ -215,7 +216,7 @@ def _create_broadcast_event(broadcast_message):
 
     dao_save_object(event)
 
-    if not broadcast_message.stubbed:
+    if not broadcast_message.stubbed or current_app.config['NOTIFY_ENVIRONMENT'] == 'preview':
         send_broadcast_event.apply_async(
             kwargs={'broadcast_event_id': str(event.id)},
             queue=QueueNames.BROADCASTS

app/broadcast_message/translators.py

@@ -0,0 +1,31 @@
+from bs4 import BeautifulSoup
+
+
+def cap_xml_to_dict(cap_xml):
+    # This function assumes that it’s being passed valid CAP XML
+    cap = BeautifulSoup(cap_xml, "xml")
+    return {
+        "reference": cap.alert.identifier.text,
+        "category": cap.alert.info.category.text,
+        "expires": cap.alert.info.expires.text,
+        "content": cap.alert.info.description.text,
+        "areas": [
+            {
+                "name": area.areaDesc.text,
+                "polygons": [
+                    cap_xml_polygon_to_list(polygon.text)
+                    for polygon in area.find_all('polygon')
+                ]
+            }
+            for area in cap.alert.info.find_all('area')
+        ]
+    }
+
+
+def cap_xml_polygon_to_list(polygon_string):
+    return [
+        [
+            float(coordinate) for coordinate in pair.split(',')
+        ]
+        for pair in polygon_string.split(' ')
+    ]

app/celery/canary_tasks.py

@@ -0,0 +1,19 @@
+from flask import current_app
+from notifications_utils.statsd_decorators import statsd
+
+from app import counter, notify_celery
+
+
+@statsd(namespace="tasks")
+def canary_passing_task():
+    global counter
+    counter += 1
+
+    current_app.logger.info(f'The counter is {counter}.')
+
+
+@notify_celery.task(name="canary_failing_task", max_retries=60, default_retry_delay=30)
+@statsd(namespace="tasks")
+def canary_failing_task():
+    current_app.logger.info('Calling the failing task')
+    raise Exception

app/celery/provider_tasks.py

@@ -6,6 +6,7 @@ from app import notify_celery
 from app.config import QueueNames
 from app.clients.email import EmailClientNonRetryableException
 from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
+from app.clients.sms import SmsClientResponseException
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.delivery import send_to_providers
@@ -22,11 +23,17 @@ def deliver_sms(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_sms_to_provider(notification)
-    except Exception:
-        try:
+    except Exception as e:
+        if isinstance(e, SmsClientResponseException):
+            current_app.logger.warning(
+                "SMS notification delivery for id: {} failed".format(notification_id)
+            )
+        else:
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
+
+        try:
             if self.request.retries == 0:
                 self.retry(queue=QueueNames.RETRY, countdown=0)
             else:

app/celery/scheduled_tasks.py

@@ -204,7 +204,7 @@ def check_precompiled_letter_state():
             https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-letter-pending-virus-scan-for-90-minutes.
             Notifications: {}""".format(len(letters), sorted(letter_ids))
 
-        current_app.logger.exception(msg)
+        current_app.logger.warning(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             zendesk_client.create_ticket(
@@ -225,7 +225,7 @@ def check_templated_letter_state():
         msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
               "Notifications: {}".format(len(letters), letter_ids)
 
-        current_app.logger.exception(msg)
+        current_app.logger.warning(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             zendesk_client.create_ticket(

app/clients/cbc_proxy.py

@@ -210,6 +210,7 @@ class CBCProxyEE(CBCProxyClientBase):
             'sent': sent,
             'expires': expires,
             'language': self.infer_language_from(description),
+            'channel': 'test',
         }
         self._invoke_lambda_with_failover(payload=payload)
 
@@ -270,6 +271,7 @@ class CBCProxyThree(CBCProxyClientBase):
             'sent': sent,
             'expires': expires,
             'language': self.infer_language_from(description),
+            'channel': 'test',
         }
         self._invoke_lambda_with_failover(payload=payload)
 
@@ -329,6 +331,7 @@ class CBCProxyO2(CBCProxyClientBase):
             'sent': sent,
             'expires': expires,
             'language': self.infer_language_from(description),
+            'channel': 'test',
         }
         self._invoke_lambda_with_failover(payload=payload)
 
@@ -391,6 +394,7 @@ class CBCProxyVodafone(CBCProxyClientBase):
             'sent': sent,
             'expires': expires,
             'language': self.infer_language_from(description),
+            'channel': 'test',
         }
         self._invoke_lambda_with_failover(payload=payload)
 

app/clients/sms/firetext.py

@@ -88,7 +88,7 @@ class FiretextClient(SmsClient):
             self.statsd_client.incr("clients.firetext.success")
         else:
             self.statsd_client.incr("clients.firetext.error")
-            self.current_app.logger.error(log_message)
+            self.current_app.logger.warning(log_message)
 
     def send_sms(self, to, content, reference, sender=None):
 

app/clients/sms/mmg.py

@@ -90,7 +90,7 @@ class MMGClient(SmsClient):
             self.statsd_client.incr("clients.mmg.success")
         else:
             self.statsd_client.incr("clients.mmg.error")
-            self.current_app.logger.error(log_message)
+            self.current_app.logger.warning(log_message)
 
     def get_name(self):
         return self.name

app/v2/broadcast/__init__.py

@@ -0,0 +1,10 @@
+from flask import Blueprint
+from app.v2.errors import register_errors
+
+v2_broadcast_blueprint = Blueprint(
+    "v2_broadcast_blueprint",
+    __name__,
+    url_prefix='/v2/broadcast',
+)
+
+register_errors(v2_broadcast_blueprint)

app/v2/broadcast/broadcast_schemas.py

@@ -0,0 +1,97 @@
+post_broadcast_schema = {
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "type": "object",
+    "required": [
+        "reference",
+        "category",
+        "content",
+        "areas",
+    ],
+    "additionalProperties": False,
+    "properties": {
+        "reference": {
+            "type": [
+                "string",
+                "null",
+            ],
+        },
+        "category": {
+            "type": "string",
+            "enum": [
+                "Geo",
+                "Met",
+                "Safety",
+                "Security",
+                "Rescue",
+                "Fire",
+                "Health",
+                "Env",
+                "Transport",
+                "Infra",
+                "CBRNE",
+                "Other",
+            ],
+        },
+        "expires": {
+            "type": "string",
+            "format": "date-time",
+        },
+        "content": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 1395,
+        },
+        "web": {
+            "type": "string",
+            "format": "uri",
+        },
+        "areas": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "$ref": "#/definitions/area",
+            },
+        },
+    },
+    "definitions": {
+        "area": {
+            "type": "object",
+            "required": [
+                "name",
+                "polygons",
+            ],
+            "additionalProperties": False,
+            "properties": {
+                "name": {
+                    "type": "string",
+                },
+                "polygons": {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "oneOf": [
+                            {
+                                "$ref": "#/definitions/polygon",
+                            },
+                        ],
+                    },
+                },
+            },
+        },
+        "polygon": {
+            "type": "array",
+            "minItems": 4,
+            "items": {
+                "$ref": "#/definitions/coordinatePair",
+            },
+        },
+        "coordinatePair": {
+            "type": "array",
+            "items": {
+                "type": "number"
+            },
+            "minItems": 2,
+            "maxItems": 2,
+        },
+    },
+}

app/v2/broadcast/post_broadcast.py

@@ -0,0 +1,71 @@
+from itertools import chain
+from flask import current_app, jsonify, request
+from notifications_utils.polygons import Polygons
+from app import authenticated_service, api_user
+from app.broadcast_message.translators import cap_xml_to_dict
+from app.dao.dao_utils import dao_save_object
+from app.notifications.validators import check_service_has_permission
+from app.models import BROADCAST_TYPE, BroadcastMessage, BroadcastStatusType
+from app.schema_validation import validate
+from app.v2.broadcast import v2_broadcast_blueprint
+from app.v2.broadcast.broadcast_schemas import post_broadcast_schema
+from app.v2.errors import BadRequestError
+from app.xml_schemas import validate_xml
+
+
+@v2_broadcast_blueprint.route("", methods=['POST'])
+def create_broadcast():
+
+    check_service_has_permission(
+        BROADCAST_TYPE,
+        authenticated_service.permissions,
+    )
+
+    if request.content_type != 'application/cap+xml':
+        raise BadRequestError(
+            message=f'Content type {request.content_type} not supported',
+            status_code=415,
+        )
+
+    cap_xml = request.get_data(as_text=True)
+
+    if not validate_xml(cap_xml, 'CAP-v1.2.xsd'):
+        raise BadRequestError(
+            message=f'Request data is not valid CAP XML',
+            status_code=400,
+        )
+
+    broadcast_json = cap_xml_to_dict(cap_xml)
+
+    validate(broadcast_json, post_broadcast_schema)
+
+    polygons = Polygons(list(chain.from_iterable((
+        area['polygons'] for area in broadcast_json['areas']
+    ))))
+
+    broadcast_message = BroadcastMessage(
+        service_id=authenticated_service.id,
+        content=broadcast_json['content'],
+        reference=broadcast_json['reference'],
+        areas={
+            'areas': [
+                area['name'] for area in broadcast_json['areas']
+            ],
+            'simple_polygons': polygons.smooth.simplify.as_coordinate_pairs_long_lat,
+        },
+        status=BroadcastStatusType.PENDING_APPROVAL,
+        api_key_id=api_user.id,
+        # The client may pass in broadcast_json['expires'] but it’s
+        # simpler for now to ignore it and have the rules around expiry
+        # for broadcasts created with the API match those created from
+        # the admin app
+    )
+
+    dao_save_object(broadcast_message)
+
+    current_app.logger.info(
+        f'Broadcast message {broadcast_message.id} created for service '
+        f'{authenticated_service.id} with reference {broadcast_json["reference"]}'
+    )
+
+    return jsonify(broadcast_message.serialize()), 201

app/xml_schemas/CAP-v1.2.xsd

@@ -0,0 +1,218 @@
+<?xml version = "1.0" encoding = "UTF-8"?>
+<!-- Copyright OASIS Open 2010 All Rights Reserved -->
+<schema xmlns = "http://www.w3.org/2001/XMLSchema"
+   targetNamespace = "urn:oasis:names:tc:emergency:cap:1.2"
+   xmlns:cap = "urn:oasis:names:tc:emergency:cap:1.2"
+   xmlns:xs = "http://www.w3.org/2001/XMLSchema"
+   elementFormDefault = "qualified"
+   attributeFormDefault = "unqualified"
+   version = "1.2">
+  <element name = "alert">
+    <annotation>
+      <documentation>CAP Alert Message (version 1.2)</documentation>
+    </annotation>
+    <complexType>
+      <sequence>
+        <element name = "identifier" type = "xs:string"/>
+        <element name = "sender" type = "xs:string"/>
+        <element name = "sent">
+          <simpleType>
+            <restriction base = "xs:dateTime">
+              <pattern value = "\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d[-,+]\d\d:\d\d"/>
+            </restriction>
+          </simpleType>
+        </element>
+        <element name = "status">
+          <simpleType>
+            <restriction base = "xs:string">
+              <enumeration value = "Actual"/>
+              <enumeration value = "Exercise"/>
+              <enumeration value = "System"/>
+              <enumeration value = "Test"/>
+              <enumeration value = "Draft"/>
+            </restriction>
+          </simpleType>
+        </element>
+        <element name = "msgType">
+          <simpleType>
+            <restriction base = "xs:string">
+              <enumeration value = "Alert"/>
+              <enumeration value = "Update"/>
+              <enumeration value = "Cancel"/>
+              <enumeration value = "Ack"/>
+              <enumeration value = "Error"/>
+            </restriction>
+          </simpleType>
+        </element>
+        <element name = "source" type = "xs:string" minOccurs = "0"/>
+        <element name = "scope">
+          <simpleType>
+            <restriction base = "xs:string">
+              <enumeration value = "Public"/>
+              <enumeration value = "Restricted"/>
+              <enumeration value = "Private"/>
+            </restriction>
+          </simpleType>
+        </element>
+        <element name = "restriction" type = "xs:string" minOccurs = "0"/>
+        <element name = "addresses" type = "xs:string" minOccurs = "0"/>
+        <element name = "code" type = "xs:string" minOccurs = "0" maxOccurs = "unbounded"/>
+        <element name = "note" type = "xs:string" minOccurs = "0"/>
+        <element name = "references" type = "xs:string" minOccurs = "0"/>
+        <element name = "incidents" type = "xs:string" minOccurs = "0"/>
+        <element name = "info" minOccurs = "0" maxOccurs = "unbounded">
+          <complexType>
+            <sequence>
+              <element name = "language" type = "xs:language" default = "en-US" minOccurs = "0"/>
+              <element name = "category" maxOccurs = "unbounded">
+                <simpleType>
+                  <restriction base = "xs:string">
+                    <enumeration value = "Geo"/>
+                    <enumeration value = "Met"/>
+                    <enumeration value = "Safety"/>
+                    <enumeration value = "Security"/>
+                    <enumeration value = "Rescue"/>
+                    <enumeration value = "Fire"/>
+                    <enumeration value = "Health"/>
+                    <enumeration value = "Env"/>
+                    <enumeration value = "Transport"/>
+                    <enumeration value = "Infra"/>
+                    <enumeration value = "CBRNE"/>
+                    <enumeration value = "Other"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "event" type = "xs:string"/>
+              <element name = "responseType" minOccurs = "0" maxOccurs = "unbounded">
+                <simpleType>
+                  <restriction base = "xs:string">
+                    <enumeration value = "Shelter"/>
+                    <enumeration value = "Evacuate"/>
+                    <enumeration value = "Prepare"/>
+                    <enumeration value = "Execute"/>
+                    <enumeration value = "Avoid"/>
+                    <enumeration value = "Monitor"/>
+                    <enumeration value = "Assess"/>
+                    <enumeration value = "AllClear"/>
+                    <enumeration value = "None"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "urgency">
+                <simpleType>
+                  <restriction base = "xs:string">
+                    <enumeration value = "Immediate"/>
+                    <enumeration value = "Expected"/>
+                    <enumeration value = "Future"/>
+                    <enumeration value = "Past"/>
+                    <enumeration value = "Unknown"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "severity">
+                <simpleType>
+                  <restriction base = "xs:string">
+                    <enumeration value = "Extreme"/>
+                    <enumeration value = "Severe"/>
+                    <enumeration value = "Moderate"/>
+                    <enumeration value = "Minor"/>
+                    <enumeration value = "Unknown"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "certainty">
+                <simpleType>
+                  <restriction base = "xs:string">
+                    <enumeration value = "Observed"/>
+                    <enumeration value = "Likely"/>
+                    <enumeration value = "Possible"/>
+                    <enumeration value = "Unlikely"/>
+                    <enumeration value = "Unknown"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "audience" type = "xs:string" minOccurs = "0"/>
+              <element name = "eventCode" minOccurs = "0" maxOccurs = "unbounded">
+                <complexType>
+                  <sequence>
+                    <element ref = "cap:valueName"/>
+                    <element ref = "cap:value"/>
+                  </sequence>
+                </complexType>
+              </element>
+              <element name = "effective" minOccurs = "0">
+                <simpleType>
+                  <restriction base = "xs:dateTime">
+                    <pattern value = "\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d[-,+]\d\d:\d\d"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "onset"  minOccurs = "0">
+                <simpleType>
+                  <restriction base = "xs:dateTime">
+                    <pattern value = "\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d[-,+]\d\d:\d\d"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "expires"  minOccurs = "0">
+                <simpleType>
+                  <restriction base = "xs:dateTime">
+                    <pattern value = "\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d[-,+]\d\d:\d\d"/>
+                  </restriction>
+                </simpleType>
+              </element>
+              <element name = "senderName" type = "xs:string" minOccurs = "0"/>
+              <element name = "headline" type = "xs:string" minOccurs = "0"/>
+              <element name = "description" type = "xs:string" minOccurs = "0"/>
+              <element name = "instruction" type = "xs:string" minOccurs = "0"/>
+              <element name = "web" type = "xs:anyURI" minOccurs = "0"/>
+              <element name = "contact" type = "xs:string" minOccurs = "0"/>
+              <element name = "parameter" minOccurs = "0" maxOccurs = "unbounded">
+                <complexType>
+                  <sequence>
+                    <element ref = "cap:valueName"/>
+                    <element ref = "cap:value"/>
+                  </sequence>
+                </complexType>
+              </element>
+              <element name = "resource" minOccurs = "0" maxOccurs = "unbounded">
+                <complexType>
+                  <sequence>
+                    <element name = "resourceDesc" type = "xs:string"/>
+                    <element name = "mimeType" type = "xs:string"/>
+                    <element name = "size" type = "xs:integer" minOccurs = "0"/>
+                    <element name = "uri" type = "xs:anyURI" minOccurs = "0"/>
+                    <element name = "derefUri" type = "xs:string" minOccurs = "0"/>
+                    <element name = "digest" type = "xs:string" minOccurs = "0"/>
+                  </sequence>
+                </complexType>
+              </element>
+              <element name = "area" minOccurs = "0" maxOccurs = "unbounded">
+                <complexType>
+                  <sequence>
+                    <element name = "areaDesc" type = "xs:string"/>
+                    <element name = "polygon" type = "xs:string" minOccurs = "0" maxOccurs = "unbounded"/>
+                    <element name = "circle" type = "xs:string" minOccurs = "0" maxOccurs = "unbounded"/>
+                    <element name = "geocode" minOccurs = "0" maxOccurs = "unbounded">
+                      <complexType>
+                        <sequence>
+                          <element ref = "cap:valueName"/>
+                          <element ref = "cap:value"/>
+                        </sequence>
+                      </complexType>
+                    </element>
+                    <element name = "altitude" type = "xs:decimal" minOccurs = "0"/>
+                    <element name = "ceiling" type = "xs:decimal" minOccurs = "0"/>
+                  </sequence>
+                </complexType>
+              </element>
+            </sequence>
+          </complexType>
+        </element>
+        <any minOccurs = "0" maxOccurs = "unbounded" namespace = "http://www.w3.org/2000/09/xmldsig#" processContents = "lax"/>
+      </sequence>
+    </complexType>
+  </element>
+  <element name = "valueName" type = "xs:string"/>
+  <element name = "value" type = "xs:string"/>
+</schema>

app/xml_schemas/__init__.py

@@ -0,0 +1,19 @@
+from lxml import etree
+from pathlib import Path
+
+
+def validate_xml(document, schema_file_name):
+
+    path = Path(__file__).resolve().parent / schema_file_name
+    contents = path.read_text()
+
+    schema_root = etree.XML(contents.encode('utf-8'))
+    schema = etree.XMLSchema(schema_root)
+    parser = etree.XMLParser(schema=schema)
+
+    try:
+        etree.fromstring(document, parser)
+    except etree.XMLSyntaxError:
+        return False
+
+    return True

manifest.yml.j2

@@ -68,6 +68,7 @@
   'notify-delivery-worker-receipts': {},
   'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
   'notify-delivery-worker-save-api-notifications': {'disk_quota': '2G'},
+  'notify-delivery-worker-canary': {'disk_quota': '2G'},
 } -%}
 
 {%- set app = app_vars[CF_APP] -%}

migrations/versions/0341_new_letter_rates.py

@@ -0,0 +1,62 @@
+"""
+
+Revision ID: 0341_new_letter_rates
+Revises: 0340_stub_training_broadcasts
+Create Date: 2021-01-27 11:58:21.393227
+
+"""
+import itertools
+import uuid
+from datetime import datetime
+
+from alembic import op
+from sqlalchemy.sql import text
+
+from app.models import LetterRate
+
+
+revision = '0341_new_letter_rates'
+down_revision = '0340_stub_training_broadcasts'
+
+
+CHANGEOVER_DATE = datetime(2021, 2, 1, 0, 0)
+
+
+def get_new_rate(sheet_count, post_class):
+    base_prices = {
+        'second': 34,
+        'first': 64,
+        'europe': 81,
+        'rest-of-world': 81,
+    }
+    multiplier = 5 if post_class in ('first', 'second') else 8
+
+    return (base_prices[post_class] + (multiplier * sheet_count)) / 100.0
+
+
+def upgrade():
+    conn = op.get_bind()
+    conn.execute(text("UPDATE letter_rates SET end_date = :start WHERE end_date IS NULL"), start=CHANGEOVER_DATE)
+
+    op.bulk_insert(LetterRate.__table__, [
+        {
+            'id': uuid.uuid4(),
+            'start_date': CHANGEOVER_DATE,
+            'end_date': None,
+            'sheet_count': sheet_count,
+            'rate': get_new_rate(sheet_count, post_class),
+            'crown': crown,
+            'post_class': post_class,
+        }
+        for sheet_count, crown, post_class in itertools.product(
+            range(1, 6),
+            [True, False],
+            ['first', 'second', 'europe', 'rest-of-world']
+        )
+    ])
+
+def downgrade():
+    # Make sure you've thought about billing implications etc before downgrading!
+    conn = op.get_bind()
+    conn.execute(text("DELETE FROM letter_rates WHERE start_date = :start"), start=CHANGEOVER_DATE)
+    conn.execute(text("UPDATE letter_rates SET end_date = NULL WHERE end_date = :start"), start=CHANGEOVER_DATE)

requirements-app.txt

@@ -23,13 +23,15 @@ SQLAlchemy==1.3.22
 strict-rfc3339==0.7
 rfc3987==1.3.8
 cachetools==4.2.0
+beautifulsoup4==4.9.3
+lxml==4.6.2
 
 notifications-python-client==5.7.1
 
 # PaaS
 awscli-cwlogs==1.4.6
 
-git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9
+git+https://github.com/alphagov/notifications-utils.git@43.8.1#egg=notifications-utils==43.8.1
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
 prometheus-client==0.9.0

requirements.txt

@@ -25,31 +25,33 @@ SQLAlchemy==1.3.22
 strict-rfc3339==0.7
 rfc3987==1.3.8
 cachetools==4.2.0
+beautifulsoup4==4.9.3
+lxml==4.6.2
 
 notifications-python-client==5.7.1
 
 # PaaS
 awscli-cwlogs==1.4.6
 
-git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9
+git+https://github.com/alphagov/notifications-utils.git@43.8.1#egg=notifications-utils==43.8.1
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
 prometheus-client==0.9.0
 gds-metrics==0.2.4
 
 ## The following requirements were added by pip freeze:
-alembic==1.4.3
+alembic==1.5.2
 amqp==1.4.9
 anyjson==0.3.3
 attrs==20.3.0
-awscli==1.18.206
+awscli==1.18.222
 bcrypt==3.2.0
 billiard==3.3.0.23
 bleach==3.2.1
 blinker==1.4
 boto==2.49.0
-boto3==1.16.46
-botocore==1.19.46
+boto3==1.16.62
+botocore==1.19.62
 certifi==2020.12.5
 chardet==4.0.0
 click==7.1.2
@@ -60,19 +62,19 @@ flask-redis==0.4.0
 future==0.18.2
 geojson==2.5.0
 govuk-bank-holidays==0.8
-greenlet==0.4.17
+greenlet==1.0.0
 idna==2.10
-importlib-metadata==3.3.0
+importlib-metadata==3.4.0
 Jinja2==2.11.2
 jmespath==0.10.0
 kombu==3.0.37
-Mako==1.1.3
+Mako==1.1.4
 MarkupSafe==1.1.1
 mistune==0.8.4
 monotonic==1.5
 orderedset==2.0.3
 packaging==20.8
-phonenumbers==8.12.15
+phonenumbers==8.12.17
 pyasn1==0.4.8
 pycparser==2.20
 pyparsing==2.4.7
@@ -86,12 +88,14 @@ PyYAML==5.3.1
 redis==3.5.3
 requests==2.25.1
 rsa==4.5
-s3transfer==0.3.3
+s3transfer==0.3.4
+Shapely==1.7.1
 six==1.15.0
 smartypants==2.0.1
+soupsieve==2.1
 statsd==3.3.0
 typing-extensions==3.7.4.3
-urllib3==1.26.2
+urllib3==1.26.3
 webencodings==0.5.1
 Werkzeug==1.0.1
 zipp==3.4.0

scripts/paas_app_wrapper.sh

@@ -57,6 +57,10 @@ case $NOTIFY_APP_NAME in
     exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
     -Q save-api-email-tasks,save-api-sms-tasks 2> /dev/null
     ;;
+  delivery-worker-canary)
+      exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q canary-a,canary-b 2> /dev/null
+    ;;
   delivery-celery-beat)
     exec scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
     ;;

tests/app/celery/test_provider_tasks.py

@@ -5,6 +5,7 @@ from celery.exceptions import MaxRetriesExceededError
 import app
 from app.celery import provider_tasks
 from app.celery.provider_tasks import deliver_sms, deliver_email
+from app.clients.sms import SmsClientResponseException
 from app.clients.email import EmailClientNonRetryableException
 from app.clients.email.aws_ses import AwsSesClientException, AwsSesClientThrottlingSendRateException
 from app.exceptions import NotificationTechnicalFailureException
@@ -37,6 +38,68 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task
     app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
 
 
+def test_send_sms_should_not_switch_providers_on_non_provider_failure(
+    sample_notification,
+    mocker
+):
+    mocker.patch(
+        'app.delivery.send_to_providers.send_sms_to_provider',
+        side_effect=Exception("Non Provider Exception")
+    )
+    mock_dao_reduce_sms_provider_priority = mocker.patch(
+        'app.delivery.send_to_providers.dao_reduce_sms_provider_priority'
+    )
+    mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
+
+    deliver_sms(sample_notification.id)
+
+    assert mock_dao_reduce_sms_provider_priority.called is False
+
+
+def test_should_retry_and_log_warning_if_SmsClientResponseException_for_deliver_sms_task(sample_notification, mocker):
+    mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=SmsClientResponseException("something went wrong"))
+    mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
+    mock_logger_warning = mocker.patch('app.celery.tasks.current_app.logger.warning')
+
+    deliver_sms(sample_notification.id)
+
+    assert provider_tasks.deliver_sms.retry.called is True
+    assert sample_notification.status == 'created'
+    assert mock_logger_warning.called
+
+
+def test_should_retry_and_log_exception_for_non_SmsClientResponseException_exceptions_for_deliver_sms_task(
+    sample_notification, mocker
+):
+    mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("something went wrong"))
+    mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
+    mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
+
+    deliver_sms(sample_notification.id)
+
+    assert provider_tasks.deliver_sms.retry.called is True
+    assert sample_notification.status == 'created'
+    assert mock_logger_exception.called
+
+
+def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(sample_notification, mocker):
+    mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
+    mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())
+    mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
+
+    with pytest.raises(NotificationTechnicalFailureException) as e:
+        deliver_sms(sample_notification.id)
+    assert str(sample_notification.id) in str(e.value)
+
+    provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
+
+    assert sample_notification.status == 'technical-failure'
+    assert mock_logger_exception.called
+
+
+# end of deliver_sms task tests, now deliver_email task tests
+
+
 def test_should_call_send_email_to_provider_from_deliver_email_task(
         sample_notification,
         mocker):
@@ -57,21 +120,6 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta
     app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
 
 
-# DO THESE FOR THE 4 TYPES OF TASK
-
-def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(sample_notification, mocker):
-    mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
-    mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())
-
-    with pytest.raises(NotificationTechnicalFailureException) as e:
-        deliver_sms(sample_notification.id)
-    assert str(sample_notification.id) in str(e.value)
-
-    provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
-
-    assert sample_notification.status == 'technical-failure'
-
-
 @pytest.mark.parametrize(
     'exception_class', [
         Exception(),
@@ -106,7 +154,7 @@ def test_should_technical_error_and_not_retry_if_EmailClientNonRetryableExceptio
     assert sample_notification.status == 'technical-failure'
 
 
-def test_should_retry_and_log_exception(sample_notification, mocker):
+def test_should_retry_and_log_exception_for_deliver_email_task(sample_notification, mocker):
     error_response = {
         'Error': {
             'Code': 'SomeError',
@@ -149,21 +197,3 @@ def test_if_ses_send_rate_throttle_then_should_retry_and_log_warning(sample_noti
     assert sample_notification.status == 'created'
     assert not mock_logger_exception.called
     assert mock_logger_warning.called
-
-
-def test_send_sms_should_not_switch_providers_on_non_provider_failure(
-    sample_notification,
-    mocker
-):
-    mocker.patch(
-        'app.delivery.send_to_providers.send_sms_to_provider',
-        side_effect=Exception("Non Provider Exception")
-    )
-    mock_dao_reduce_sms_provider_priority = mocker.patch(
-        'app.delivery.send_to_providers.dao_reduce_sms_provider_priority'
-    )
-    mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
-
-    deliver_sms(sample_notification.id)
-
-    assert mock_dao_reduce_sms_provider_priority.called is False

tests/app/celery/test_scheduled_tasks.py

@@ -320,7 +320,7 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
 
 @freeze_time("2019-05-30 14:00:00")
 def test_check_precompiled_letter_state(mocker, sample_letter_template):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
     create_notification(template=sample_letter_template,
@@ -357,7 +357,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
 
 @freeze_time("2019-05-30 14:00:00")
 def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
     noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
@@ -382,7 +382,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
 
 @freeze_time("2019-01-30 14:00:00")
 def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
 
     noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))

tests/app/clients/test_cbc_proxy.py

@@ -133,6 +133,7 @@ def test_cbc_proxy_one_2_many_create_and_send_invokes_function(
     assert payload['sent'] == sent
     assert payload['expires'] == expires
     assert payload['language'] == expected_language
+    assert payload['channel'] == 'test'
 
 
 @pytest.mark.parametrize('cbc', ['ee', 'three', 'o2'])
@@ -249,6 +250,7 @@ def test_cbc_proxy_vodafone_create_and_send_invokes_function(
     assert payload['sent'] == sent
     assert payload['expires'] == expires
     assert payload['language'] == expected_language
+    assert payload['channel'] == 'test'
 
 
 def test_cbc_proxy_vodafone_cancel_invokes_function(mocker, cbc_proxy_vodafone):

tests/app/notifications/test_process_notification.py

@@ -362,7 +362,8 @@ def test_simulated_recipient(notify_api, to_address, notification_type, expected
 @pytest.mark.parametrize('recipient, expected_international, expected_prefix, expected_units', [
     ('7900900123', False, '44', 1),  # UK
     ('+447900900123', False, '44', 1),  # UK
-    ('07700900222', True, '44', 1),  # UK (Jersey)
+    ('07700910222', True, '44', 1),  # UK (Jersey)
+    ('07700900222', False, '44', 1),  # TV number
     ('73122345678', True, '7', 1),  # Russia
     ('360623400400', True, '36', 3)]  # Hungary
 )

tests/app/v2/broadcast/sample_cap_xml_documents.py

@@ -0,0 +1,34 @@
+WAINFLEET = """
+    <alert xmlns="urn:oasis:names:tc:emergency:cap:1.2">
+        <identifier>50385fcb0ab7aa447bbd46d848ce8466E</identifier>
+        <sender>www.gov.uk/environment-agency</sender>
+        <sent>2020-02-16T23:01:13-00:00</sent>
+        <status>Actual</status>
+        <msgType>Update</msgType>
+        <source>Flood warning service</source>
+        <scope>Public</scope>
+        <references>www.gov.uk/environment-agency,4f6d28b10ab7aa447bbd46d85f1e9effE,2020-02-16T19:20:03+00:00</references>
+        <info>
+            <language>en-GB</language>
+            <category>Met</category>
+            <event>053/055 Issue Severe Flood Warning EA</event>
+            <urgency>Immediate</urgency>
+            <severity>Severe</severity>
+            <certainty>Likely</certainty>
+            <expires>2020-02-26T23:01:14-00:00</expires>
+            <senderName>Environment Agency</senderName>
+            <description>A severe flood warning has been issued. Storm Dennis has resulted in significant rainfall in the Steeping River catchment with several bands of heavy rain passing through the area during today (Sunday 16 Feb). River levels along the Steeping River and the Wainfleet relief channel are expected to be similar to those in June 2019. This could result in flood embankments being overtopped. Should this happen, there is an increased risk that flood embankments could breach. It is expected that peak levels along the Steeping River in Wainfleet will be between midnight and 3am tonight. A multi-agency meeting is taking place this evening. Further messages will be issued should this be required. Do not walk on flood embankments and avoid riverside paths. Our staff are out in the area to check the flood defences and assist the emergency services and council. We will be closely monitoring the situation throughout the night. </description>
+            <instruction># To check the latest information for your area - Visit [GOV.UK](https://flood-warning-information.service.gov.uk) to see the current flood warnings, view river and sea levels or check the 5-day flood risk forecast: https://flood-warning-information.service.gov.uk - Follow [@EnvAgency](https://twitter.com/EnvAgency) and [#floodaware](https://twitter.com/hashtag/floodaware) on Twitter. - Tune into weather, news and travel bulletins on local television and radio. - For access to flood warning information offline call Floodline on 0345 988 1188 using quickdial code: 307052. # What you should consider doing now - Call 999 if you are in immediate danger. - Co-operate with the emergency services and evacuate your property if told to do so. Most evacuation centres will let you bring your pets. - Act on your flood plan if you have one. - Move your family and pets to a safe place with a means of escape. - Use flood protection equipment (such as flood barriers, air brick covers and pumps) to protect your property. Unless you have proper equipment do not waste valuable time trying to keep the water out. - Move important items upstairs or to a safe place in your property, starting with cherished items of personal value that you will not be able to replace (such as family photographs). Next move valuables (such as computers), movable furniture and furnishings. - You may need to leave your property, so pack a bag with enough items for a few nights away. Include essential items including a torch with spare batteries, mobile phone and charger, warm clothes, home insurance information, water, food, first aid kit and any prescription medicines or baby care items you may need. - Turn off gas, electricity and water mains supplies before flood water starts to enter your property. Never touch an electrical switch if you are standing in water. - If it is safe to do so, make sure neighbours are aware of the situation and offer help to anyone who may need it. - Avoid walking, cycling or driving through flood water - 30 cm of fast-flowing water can move a car and 6 inches can knock an adult off their feet. - Flood water is dangerous and may be polluted. Wash your hands thoroughly if you’ve been in contact with it. ##### Businesses - Act on your business flood plan if you have one. - Move your staff and customers to a safe place with a means of escape. - Move stock and other valuable items upstairs or to a safe place in your building. For media enquiries please contact our media teams: https://www.gov.uk/government/organisations/environment-agency/about/media-enquiries </instruction>
+            <web>https://flood-warning-information.service.gov.uk</web>
+            <contact>0345 988 1188</contact>
+            <area>
+                <areaDesc>River Steeping in Wainfleet All Saints</areaDesc>
+                <polygon>53.10569,0.24453 53.10593,0.24430 53.10601,0.24375 53.10615,0.24349 53.10629,0.24356 53.10656,0.24336 53.10697,0.24354 53.10684,0.24298 53.10694,0.24264 53.10721,0.24302 53.10752,0.24310 53.10777,0.24308 53.10805,0.24320 53.10803,0.24187 53.10776,0.24085 53.10774,0.24062 53.10702,0.24056 53.10679,0.24088 53.10658,0.24071 53.10651,0.24049 53.10656,0.24022 53.10642,0.24022 53.10632,0.24052 53.10629,0.24082 53.10612,0.24093 53.10583,0.24133 53.10564,0.24178 53.10541,0.24282 53.10569,0.24453</polygon>
+                <geocode>
+                    <valueName>TargetAreaCode</valueName>
+                    <value>053FWFSTEEP4</value>
+                </geocode>
+            </area>
+        </info>
+    </alert>
+"""

tests/app/v2/broadcast/test_post_broadcast.py

@@ -0,0 +1,131 @@
+from flask import json
+from freezegun import freeze_time
+from tests import create_authorization_header
+from unittest.mock import ANY
+from . import sample_cap_xml_documents
+
+
+def test_broadcast_for_service_without_permission_returns_400(
+    client,
+    sample_service,
+):
+    auth_header = create_authorization_header(service_id=sample_service.id)
+    response = client.post(
+        path='/v2/broadcast',
+        data='',
+        headers=[('Content-Type', 'application/json'), auth_header],
+    )
+
+    assert response.status_code == 400
+    assert response.get_json()['errors'][0]['message'] == (
+        'Service is not allowed to send broadcast messages'
+    )
+
+
+def test_valid_post_broadcast_returns_201(
+    client,
+    sample_broadcast_service,
+):
+    auth_header = create_authorization_header(service_id=sample_broadcast_service.id)
+
+    response = client.post(
+        path='/v2/broadcast',
+        data=json.dumps({
+            'content': 'This is a test',
+            'reference': 'abc123',
+            'category': 'Other',
+            'areas': [
+                {
+                    'name': 'Hackney Marshes',
+                    'polygons': [[
+                        [-0.038280487060546875, 51.55738264619775],
+                        [-0.03184318542480469, 51.553913882566754],
+                        [-0.023174285888671875, 51.55812972989382],
+                        [-0.023174285888671999, 51.55812972989999],
+                        [-0.029869079589843747, 51.56165153059717],
+                        [-0.038280487060546875, 51.55738264619775],
+                    ]],
+                },
+            ],
+        }),
+        headers=[('Content-Type', 'application/json'), auth_header],
+    )
+
+    assert response.status_code == 415
+    assert json.loads(response.get_data(as_text=True)) == {
+        'errors': [{
+            'error': 'BadRequestError',
+            'message': 'Content type application/json not supported'
+        }],
+        'status_code': 415,
+    }
+
+
+def test_valid_post_cap_xml_broadcast_returns_201(
+    client,
+    sample_broadcast_service,
+):
+    auth_header = create_authorization_header(service_id=sample_broadcast_service.id)
+
+    response = client.post(
+        path='/v2/broadcast',
+        data=sample_cap_xml_documents.WAINFLEET,
+        headers=[('Content-Type', 'application/cap+xml'), auth_header],
+    )
+
+    assert response.status_code == 201
+
+    response_json = json.loads(response.get_data(as_text=True))
+
+    assert response_json['approved_at'] is None
+    assert response_json['approved_by_id'] == None
+    assert response_json['areas'] == [
+        'River Steeping in Wainfleet All Saints'
+    ]
+    assert response_json['cancelled_at'] == None
+    assert response_json['cancelled_by_id'] == None
+    assert response_json['content'].startswith(
+        'A severe flood warning has been issued. Storm Dennis'
+    )
+    assert response_json['content'].endswith(
+        'closely monitoring the situation throughout the night. '
+    )
+    assert response_json['reference'] == '50385fcb0ab7aa447bbd46d848ce8466E'
+    assert response_json['created_at']  # datetime generated by the DB so can’t freeze it
+    assert response_json['created_by_id'] == None
+    assert response_json['finishes_at'] is None
+    assert response_json['id'] == ANY
+    assert response_json['personalisation'] is None
+    assert response_json['service_id'] == str(sample_broadcast_service.id)
+    assert len(response_json['simple_polygons']) == 1
+    assert len(response_json['simple_polygons'][0]) == 23
+    assert response_json['simple_polygons'][0][0] == [53.10561946699971, 0.2441253049430708]
+    assert response_json['simple_polygons'][0][-1] == [53.10561946699971, 0.2441253049430708]
+    assert response_json['starts_at'] is None
+    assert response_json['status'] == 'pending-approval'
+    assert response_json['template_id'] is None
+    assert response_json['template_name'] is None
+    assert response_json['template_version'] is None
+    assert response_json['updated_at'] is None
+
+
+def test_invalid_post_cap_xml_broadcast_returns_400(
+    client,
+    sample_broadcast_service,
+):
+    auth_header = create_authorization_header(service_id=sample_broadcast_service.id)
+
+    response = client.post(
+        path='/v2/broadcast',
+        data="<alert>Oh no</alert>",
+        headers=[('Content-Type', 'application/cap+xml'), auth_header],
+    )
+
+    assert response.status_code == 400
+    assert json.loads(response.get_data(as_text=True)) == {
+        'errors': [{
+            'error': 'BadRequestError',
+            'message': 'Request data is not valid CAP XML'
+        }],
+        'status_code': 400,
+    }