From c9d55039eb03b9eafed31a576133658cdeae1fa4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 20 Jan 2021 16:07:19 +0000 Subject: [PATCH] Simplify polygons before storing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’re going to let people pass in fairly complex polygons, but: - we don’t want to store massive polygons - we don’t want to pass the CBCs massive polygons So this commit adds a step to simplify the polygons before storing them. We think it’s best for us to do this because: - writing code to do polygon simplification is non-trivial, and we don’t want to make all potential integrators do it - the simplification we’ve developed is domain-specific to emergency alerting, so should throw away less information than There’s a bit more detail about how we simplify polygons in https://github.com/alphagov/notifications-admin/pull/3590/files --- app/v2/broadcast/post_broadcast.py | 9 +++-- requirements-app.txt | 2 +- requirements.txt | 11 +++--- tests/app/v2/broadcast/test_post_broadcast.py | 35 ++++++++++++++----- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/app/v2/broadcast/post_broadcast.py b/app/v2/broadcast/post_broadcast.py index 18fe37b64..dbebdd4f3 100644 --- a/app/v2/broadcast/post_broadcast.py +++ b/app/v2/broadcast/post_broadcast.py @@ -1,5 +1,6 @@ from itertools import chain from flask import jsonify, request +from notifications_utils.polygons import Polygons from app import authenticated_service, api_user from app.broadcast_message.translators import cap_xml_to_dict from app.dao.dao_utils import dao_save_object @@ -38,6 +39,10 @@ def create_broadcast(): validate(broadcast_json, post_broadcast_schema) + polygons = Polygons(list(chain.from_iterable(( + area['polygons'] for area in broadcast_json['areas'] + )))) + broadcast_message = BroadcastMessage( service_id=authenticated_service.id, content=broadcast_json['content'], @@ -46,9 +51,7 @@ def create_broadcast(): 'areas': [ area['name'] for area in broadcast_json['areas'] ], - 'simple_polygons': list(chain.from_iterable(( - area['polygons'] for area in broadcast_json['areas'] - ))) + 'simple_polygons': polygons.smooth.simplify.as_coordinate_pairs_long_lat, }, status=BroadcastStatusType.PENDING_APPROVAL, api_key_id=api_user.id, diff --git a/requirements-app.txt b/requirements-app.txt index cc017a729..51b9e660e 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -31,7 +31,7 @@ notifications-python-client==5.7.1 # PaaS awscli-cwlogs==1.4.6 -git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9 +git+https://github.com/alphagov/notifications-utils.git@43.8.0#egg=notifications-utils==43.8.0 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.9.0 diff --git a/requirements.txt b/requirements.txt index f9390d13e..d98dbfaad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,25 +33,25 @@ notifications-python-client==5.7.1 # PaaS awscli-cwlogs==1.4.6 -git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9 +git+https://github.com/alphagov/notifications-utils.git@43.8.0#egg=notifications-utils==43.8.0 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.9.0 gds-metrics==0.2.4 ## The following requirements were added by pip freeze: -alembic==1.5.0 +alembic==1.5.2 amqp==1.4.9 anyjson==0.3.3 attrs==20.3.0 -awscli==1.18.216 +awscli==1.18.219 bcrypt==3.2.0 billiard==3.3.0.23 bleach==3.2.1 blinker==1.4 boto==2.49.0 -boto3==1.16.56 -botocore==1.19.56 +boto3==1.16.59 +botocore==1.19.59 certifi==2020.12.5 chardet==4.0.0 click==7.1.2 @@ -89,6 +89,7 @@ redis==3.5.3 requests==2.25.1 rsa==4.5 s3transfer==0.3.4 +Shapely==1.7.1 six==1.15.0 smartypants==2.0.1 soupsieve==2.1 diff --git a/tests/app/v2/broadcast/test_post_broadcast.py b/tests/app/v2/broadcast/test_post_broadcast.py index e38918add..f8d289e6a 100644 --- a/tests/app/v2/broadcast/test_post_broadcast.py +++ b/tests/app/v2/broadcast/test_post_broadcast.py @@ -36,9 +36,14 @@ def test_valid_post_broadcast_returns_201( 'category': 'Other', 'areas': [ { - 'name': 'Borchester Downs', + 'name': 'Hackney Marshes', 'polygons': [[ - [1, 2], [3, 4], [5, 6], [1, 2], + [-0.038280487060546875, 51.55738264619775], + [-0.03184318542480469, 51.553913882566754], + [-0.023174285888671875, 51.55812972989382], + [-0.023174285888671999, 51.55812972989999], + [-0.029869079589843747, 51.56165153059717], + [-0.038280487060546875, 51.55738264619775], ]], }, ], @@ -53,7 +58,7 @@ def test_valid_post_broadcast_returns_201( assert response_json['approved_at'] is None assert response_json['approved_by_id'] == None assert response_json['areas'] == [ - 'Borchester Downs' + 'Hackney Marshes' ] assert response_json['cancelled_at'] == None assert response_json['cancelled_by_id'] == None @@ -65,9 +70,21 @@ def test_valid_post_broadcast_returns_201( assert response_json['id'] == ANY assert response_json['personalisation'] is None assert response_json['service_id'] == str(sample_broadcast_service.id) - assert response_json['simple_polygons'] == [ - [[1, 2], [3, 4], [5, 6], [1, 2],] - ] + assert response_json['simple_polygons'] == [[ + [-0.03817522145265898, 51.557381351011166], + [-0.03791399800364216, 51.55758039392131], + [-0.030362635618559567, 51.56141279571522], + [-0.02986997783677049, 51.56152875195115], + [-0.029379069367567606, 51.561405599957745], + [-0.023537043373602105, 51.5583323982824], + [-0.02328416546450603, 51.55813395976017], + [-0.02355422144186266, 51.557933308587664], + [-0.0313493058222969, 51.55414241384808], + [-0.031840673207720764, 51.55403463730992], + [-0.032327132941933706, 51.55416275685022], + [-0.037918974384948616, 51.55717594115094], + [-0.03817522145265898, 51.557381351011166], + ]] assert response_json['starts_at'] is None assert response_json['status'] == 'pending-approval' assert response_json['template_id'] is None @@ -113,9 +130,9 @@ def test_valid_post_cap_xml_broadcast_returns_201( assert response_json['personalisation'] is None assert response_json['service_id'] == str(sample_broadcast_service.id) assert len(response_json['simple_polygons']) == 1 - assert len(response_json['simple_polygons'][0]) == 29 - assert response_json['simple_polygons'][0][0] == [53.10569, 0.24453] - assert response_json['simple_polygons'][0][-1] == [53.10569, 0.24453] + assert len(response_json['simple_polygons'][0]) == 23 + assert response_json['simple_polygons'][0][0] == [53.10561946699971, 0.2441253049430708] + assert response_json['simple_polygons'][0][-1] == [53.10561946699971, 0.2441253049430708] assert response_json['starts_at'] is None assert response_json['status'] == 'pending-approval' assert response_json['template_id'] is None