darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-27 09:43:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:alphagov/notifications-api

Browse Source
This commit is contained in:
Rebecca Law
2020-07-13 09:57:14 +01:00
parent 7125499dd0 e6578e80d0
commit 24b9f5e02e
23 changed files with 837 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/__init__.py
View File

@@ -78,12 +78,14 @@ def create_app(application):
from app.config import configs
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
print(notify_environment)
application.config.from_object(configs[notify_environment])
application.config['NOTIFY_APP_NAME'] = application.name
init_app(application)
# Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded
metrics.init_app(application)
request_helper.init_app(application)
db.init_app(application)
migrate.init_app(application, db=db)
@@ -109,7 +111,6 @@ def create_app(application):
redis_store.init_app(application)
performance_platform_client.init_app(application)
document_download_client.init_app(application)
metrics.init_app(application)
register_blueprint(application)
register_v2_blueprints(application)
@@ -153,6 +154,7 @@ def register_blueprint(application):
from app.template_folder.rest import template_folder_blueprint
from app.letter_branding.letter_branding_rest import letter_branding_blueprint
from app.upload.rest import upload_blueprint
from app.broadcast_message.rest import broadcast_message_blueprint
service_blueprint.before_request(requires_admin_auth)
application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -238,6 +240,9 @@ def register_blueprint(application):
upload_blueprint.before_request(requires_admin_auth)
application.register_blueprint(upload_blueprint)
broadcast_message_blueprint.before_request(requires_admin_auth)
application.register_blueprint(broadcast_message_blueprint)
def register_v2_blueprints(application):
from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms

0
app/broadcast_message/__init__.py Normal file
View File

48
app/broadcast_message/broadcast_message_schema.py Normal file
View File

@@ -0,0 +1,48 @@
from app.schema_validation.definitions import uuid
from app.models import BroadcastStatusType
create_broadcast_message_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'description': 'POST create broadcast_message schema',
'type': 'object',
'title': 'Create broadcast_message',
'properties': {
'template_id': uuid,
'service_id': uuid,
'created_by': uuid,
'personalisation': {'type': 'object'},
'starts_at': {'type': 'string', 'format': 'datetime'},
'finishes_at': {'type': 'string', 'format': 'datetime'},
'areas': {"type": "array", "items": {"type": "string"}},
},
'required': ['template_id', 'service_id', 'created_by'],
'additionalProperties': False
}
update_broadcast_message_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'description': 'POST update broadcast_message schema',
'type': 'object',
'title': 'Update broadcast_message',
'properties': {
'personalisation': {'type': 'object'},
'starts_at': {'type': 'string', 'format': 'datetime'},
'finishes_at': {'type': 'string', 'format': 'datetime'},
'areas': {"type": "array", "items": {"type": "string"}},
},
'required': [],
'additionalProperties': False
}
update_broadcast_message_status_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'description': 'POST update broadcast_message status schema',
'type': 'object',
'title': 'Update broadcast_message',
'properties': {
'status': {'type': 'string', 'enum': BroadcastStatusType.STATUSES},
'created_by': uuid,
},
'required': ['status', 'created_by'],
'additionalProperties': False
}

133
app/broadcast_message/rest.py Normal file
View File

@@ -0,0 +1,133 @@
from datetime import datetime
import iso8601
from flask import Blueprint, jsonify, request, current_app
from app.config import QueueNames
from app.dao.templates_dao import dao_get_template_by_id_and_service_id
from app.dao.users_dao import get_user_by_id
from app.dao.broadcast_message_dao import (
dao_create_broadcast_message,
dao_get_broadcast_message_by_id_and_service_id,
dao_get_broadcast_messages_for_service,
dao_update_broadcast_message,
)
from app.dao.services_dao import dao_fetch_service_by_id
from app.errors import register_errors
from app.models import BroadcastMessage, BroadcastStatusType
from app.celery.broadcast_message_tasks import send_broadcast_message
from app.broadcast_message.broadcast_message_schema import (
create_broadcast_message_schema,
update_broadcast_message_schema,
update_broadcast_message_status_schema,
)
from app.schema_validation import validate
broadcast_message_blueprint = Blueprint(
'broadcast_message',
__name__,
url_prefix='/service/<uuid:service_id>/broadcast-message'
)
register_errors(broadcast_message_blueprint)
def _parse_nullable_datetime(dt):
if dt:
return iso8601.parse_date(dt).replace(tzinfo=None)
return dt
@broadcast_message_blueprint.route('', methods=['GET'])
def get_broadcast_messages_for_service(service_id):
# TODO: should this return template content/data in some way? or can we rely on them being cached admin side.
# we might need stuff like template name for showing on the dashboard.
# TODO: should this paginate or filter on dates or anything?
broadcast_messages = [o.serialize() for o in dao_get_broadcast_messages_for_service(service_id)]
return jsonify(broadcast_messages=broadcast_messages)
@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['GET'])
def get_broadcast_message(service_id, broadcast_message_id):
return jsonify(dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id).serialize())
@broadcast_message_blueprint.route('', methods=['POST'])
def create_broadcast_message(service_id):
data = request.get_json()
validate(data, create_broadcast_message_schema)
service = dao_fetch_service_by_id(data['service_id'])
user = get_user_by_id(data['created_by'])
template = dao_get_template_by_id_and_service_id(data['template_id'], data['service_id'])
broadcast_message = BroadcastMessage(
service_id=service.id,
template_id=template.id,
template_version=template.version,
personalisation=data.get('personalisation', {}),
areas=data.get('areas', []),
status=BroadcastStatusType.DRAFT,
starts_at=_parse_nullable_datetime(data.get('starts_at')),
finishes_at=_parse_nullable_datetime(data.get('finishes_at')),
created_by_id=user.id,
)
dao_create_broadcast_message(broadcast_message)
return jsonify(broadcast_message.serialize()), 201
@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['POST'])
def update_broadcast_message(service_id, broadcast_message_id):
data = request.get_json()
validate(data, update_broadcast_message_schema)
broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
if 'personalisation' in data:
broadcast_message.personalisation = data['personalisation']
if 'starts_at' in data:
broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at'])
if 'finishes_at' in data:
broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at'])
if 'areas' in data:
broadcast_message.areas = data['areas']
dao_update_broadcast_message(broadcast_message)
return jsonify(broadcast_message.serialize()), 200
@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>/status', methods=['POST'])
def update_broadcast_message_status(service_id, broadcast_message_id):
data = request.get_json()
validate(data, update_broadcast_message_status_schema)
broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
new_status = data['status']
# TODO: Restrict status transitions
# TODO: validate that the user belongs to the same service, isn't the creator, has permissions, etc
if new_status == BroadcastStatusType.BROADCASTING:
broadcast_message.approved_at = datetime.utcnow()
broadcast_message.approved_by = get_user_by_id(data['created_by'])
if new_status == BroadcastStatusType.CANCELLED:
broadcast_message.cancelled_at = datetime.utcnow()
broadcast_message.cancelled_by = get_user_by_id(data['created_by'])
broadcast_message.status = new_status
current_app.logger.info(
f'broadcast_message {broadcast_message_id} moving from {broadcast_message.status} to {new_status}'
)
dao_update_broadcast_message(broadcast_message)
if new_status == BroadcastStatusType.BROADCASTING:
send_broadcast_message.apply_async(
kwargs={'broadcast_message_id': str(broadcast_message.id)},
queue=QueueNames.NOTIFY
)
return jsonify(broadcast_message.serialize()), 200

37
app/celery/broadcast_message_tasks.py Normal file
View File

@@ -0,0 +1,37 @@
import requests
from flask import current_app
from notifications_utils.statsd_decorators import statsd
from app import notify_celery
from app.dao.broadcast_message_dao import dao_get_broadcast_message_by_id
@notify_celery.task(name="send-broadcast-message")
@statsd(namespace="tasks")
def send_broadcast_message(broadcast_message_id, provider='stub-1'):
# imports of schemas from tasks have to happen within functions to prevent
# `AttributeError: 'DummySession' object has no attribute 'query'` errors in unrelated tests
from app.schemas import template_schema
broadcast_message = dao_get_broadcast_message_by_id(broadcast_message_id)
current_app.logger.info(
f'sending broadcast_message {broadcast_message_id} '
f'status {broadcast_message.status} to {provider}'
)
payload = {
"template": template_schema.dump(broadcast_message.template).data,
"broadcast_message": broadcast_message.serialize(),
}
resp = requests.post(
f'{current_app.config["CBC_PROXY_URL"]}/broadcasts/{provider}',
json=payload
)
resp.raise_for_status()
current_app.logger.info(
f'broadcast_message {broadcast_message.id} '
f'status {broadcast_message.status} sent to {provider}'
)

67
app/celery/letters_pdf_tasks.py
View File

@@ -42,6 +42,8 @@ from app.models import (
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_VALIDATION_FAILED,
NOTIFICATION_VIRUS_SCAN_FAILED,
POSTAGE_TYPES,
RESOLVE_POSTAGE_FOR_FILE_NAME
)
from app.cronitor import cronitor
@@ -127,6 +129,7 @@ def collate_letter_pdfs_to_be_sent():
that have not yet been sent.
If run after midnight, it will collect up letters created before 5:30pm the day before.
"""
current_app.logger.info("starting collate-letter-pdfs-to-be-sent")
print_run_date = convert_utc_to_bst(datetime.utcnow())
if print_run_date.time() < LETTER_PROCESSING_DEADLINE:
print_run_date = print_run_date - timedelta(days=1)
@@ -134,41 +137,45 @@ def collate_letter_pdfs_to_be_sent():
print_run_deadline = print_run_date.replace(
hour=17, minute=30, second=0, microsecond=0
)
for postage in POSTAGE_TYPES:
current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
for i, letters in enumerate(group_letters(letters_to_print)):
filenames = [letter['Key'] for letter in letters]
for i, letters in enumerate(group_letters(letters_to_print)):
filenames = [letter['Key'] for letter in letters]
hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
# eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
date=print_run_deadline.strftime("%Y-%m-%d"),
num=i + 1,
hash=hash
)
current_app.logger.info(
'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
len(filenames),
dvla_filename,
sum(letter['Size'] for letter in letters)
hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
# eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format(
date=print_run_deadline.strftime("%Y-%m-%d"),
postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
num=i + 1,
hash=hash
)
)
notify_celery.send_task(
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
kwargs={
'filenames_to_zip': filenames,
'upload_filename': dvla_filename
},
queue=QueueNames.PROCESS_FTP,
compression='zlib'
)
current_app.logger.info(
'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
len(filenames),
dvla_filename,
sum(letter['Size'] for letter in letters)
)
)
notify_celery.send_task(
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
kwargs={
'filenames_to_zip': filenames,
'upload_filename': dvla_filename
},
queue=QueueNames.PROCESS_FTP,
compression='zlib'
)
current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}")
current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
letter_pdfs = []
for letter in letters_awaiting_sending:
try:

10
app/config.py
View File

@@ -112,6 +112,9 @@ class Config(object):
# Antivirus
ANTIVIRUS_ENABLED = True
# Broadcast Messaging
CBC_PROXY_URL = None
###########################
# Default config values ###
###########################
@@ -393,6 +396,8 @@ class Development(Config):
API_HOST_NAME = "http://localhost:6011"
API_RATE_LIMIT_ENABLED = True
CBC_PROXY_URL = 'http://localhost:8080'
class Test(Development):
NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
@@ -436,6 +441,8 @@ class Test(Development):
FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
CBC_PROXY_URL = 'http://test-cbc-proxy'
MMG_URL = 'https://example.com/mmg'
FIRETEXT_URL = 'https://example.com/firetext'
@@ -452,6 +459,7 @@ class Preview(Config):
INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital'
FROM_NUMBER = 'preview'
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = False
@@ -469,6 +477,7 @@ class Staging(Config):
INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital'
FROM_NUMBER = 'stage'
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = True
@@ -487,6 +496,7 @@ class Live(Config):
INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital'
FROM_NUMBER = 'GOVUK'
PERFORMANCE_PLATFORM_ENABLED = True
API_RATE_LIMIT_ENABLED = True

30
app/dao/broadcast_message_dao.py Normal file
View File

@@ -0,0 +1,30 @@
from app import db
from app.models import BroadcastMessage
from app.dao.dao_utils import transactional
@transactional
def dao_create_broadcast_message(broadcast_message):
db.session.add(broadcast_message)
@transactional
def dao_update_broadcast_message(broadcast_message):
db.session.add(broadcast_message)
def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id):
return BroadcastMessage.query.filter(
BroadcastMessage.id == broadcast_message_id,
BroadcastMessage.service_id == service_id
).one()
def dao_get_broadcast_message_by_id(broadcast_message_id):
return BroadcastMessage.query.get(broadcast_message_id)
def dao_get_broadcast_messages_for_service(service_id):
return BroadcastMessage.query.filter(
BroadcastMessage.service_id == service_id
).order_by(BroadcastMessage.created_at)

5
app/dao/notifications_dao.py
View File

@@ -733,7 +733,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
return notifications
def dao_get_letters_to_be_printed(print_run_deadline):
def dao_get_letters_to_be_printed(print_run_deadline, postage):
"""
Return all letters created before the print run deadline that have not yet been sent
"""
@@ -741,7 +741,8 @@ def dao_get_letters_to_be_printed(print_run_deadline):
Notification.created_at < convert_bst_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL
Notification.key_type == KEY_TYPE_NORMAL,
Notification.postage == postage,
).order_by(
Notification.created_at
).all()

44
app/models.py
View File

@@ -2174,7 +2174,7 @@ class BroadcastMessage(db.Model):
{}
)
id = db.Column(UUID(as_uuid=True), primary_key=True)
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'))
service = db.relationship('Service', backref='broadcast_messages')
@@ -2184,6 +2184,8 @@ class BroadcastMessage(db.Model):
template = db.relationship('TemplateHistory', backref='broadcast_messages')
_personalisation = db.Column(db.String, nullable=True)
# defaults to empty list
areas = db.Column(JSONB(none_as_null=True), nullable=False, default=list)
status = db.Column(
db.String,
@@ -2197,7 +2199,7 @@ class BroadcastMessage(db.Model):
finishes_at = db.Column(db.DateTime, nullable=True) # isn't updated if user cancels
# these times correspond to when
created_at = db.Column(db.DateTime, nullable=False)
created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
approved_at = db.Column(db.DateTime, nullable=True)
cancelled_at = db.Column(db.DateTime, nullable=True)
updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
@@ -2209,3 +2211,41 @@ class BroadcastMessage(db.Model):
created_by = db.relationship('User', foreign_keys=[created_by_id])
approved_by = db.relationship('User', foreign_keys=[approved_by_id])
cancelled_by = db.relationship('User', foreign_keys=[cancelled_by_id])
@property
def personalisation(self):
if self._personalisation:
return encryption.decrypt(self._personalisation)
return {}
@personalisation.setter
def personalisation(self, personalisation):
self._personalisation = encryption.encrypt(personalisation or {})
def serialize(self):
return {
'id': str(self.id),
'service_id': str(self.service_id),
'template_id': str(self.template_id),
'template_version': self.template_version,
'template_name': self.template.name,
'personalisation': self.personalisation,
'areas': self.areas,
'status': self.status,
'starts_at': self.starts_at.strftime(DATETIME_FORMAT) if self.starts_at else None,
'finishes_at': self.finishes_at.strftime(DATETIME_FORMAT) if self.finishes_at else None,
'created_at': self.created_at.strftime(DATETIME_FORMAT) if self.created_at else None,
'approved_at': self.approved_at.strftime(DATETIME_FORMAT) if self.approved_at else None,
'cancelled_at': self.cancelled_at.strftime(DATETIME_FORMAT) if self.cancelled_at else None,
'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None,
'created_by_id': str(self.created_by_id),
'approved_by_id': str(self.approved_by_id),
'cancelled_by_id': str(self.cancelled_by_id),
}

10
app/notifications/process_notifications.py
View File

@@ -150,17 +150,13 @@ def persist_notification(
notification.postage = postage or template_postage
notification.normalised_to = ''.join(notification.to.split()).lower()
# Get service attributes before the commit
service_in_trial_mode = service.restricted
service_id = service.id
# if simulated create a Notification model to return but do not persist the Notification to the dB
if not simulated:
dao_create_notification(notification)
# Only keep track of the daily limit for trial mode services.
if service_in_trial_mode and key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service_id)):
redis_store.incr(redis.daily_limit_cache_key(service_id))
if service.restricted and key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service.id)):
redis_store.incr(redis.daily_limit_cache_key(service.id))
current_app.logger.info(
"{} {} created at {}".format(notification_type, notification_id, notification_created_at)

11
app/schema_validation/__init__.py
View File

@@ -54,6 +54,17 @@ def validate_schema_date_with_hour(instance):
return True
@format_checker.checks('datetime', raises=ValidationError)
def validate_schema_datetime(instance):
if isinstance(instance, str):
try:
iso8601.parse_date(instance)
except ParseError:
raise ValidationError("datetime format is invalid. It must be a valid ISO8601 date time format, "
"https://en.wikipedia.org/wiki/ISO_8601")
return True
def validate(json_to_validate, schema):
validator = Draft7Validator(schema, format_checker=format_checker)
errors = list(validator.iter_errors(json_to_validate))

1
app/serialised_models.py
View File

@@ -47,7 +47,6 @@ class SerialisedTemplate(SerialisedModel):
'subject',
'template_type',
'version',
'broadcast_data',
}
@classmethod

19
app/v2/notifications/post_notifications.py
View File

@@ -149,7 +149,6 @@ def post_notification(notification_type):
notification = process_sms_or_email_notification(
form=form,
notification_type=notification_type,
api_key=api_user,
template=template,
template_with_content=template_with_content,
template_process_type=template.process_type,
@@ -164,7 +163,6 @@ def process_sms_or_email_notification(
*,
form,
notification_type,
api_key,
template,
template_with_content,
template_process_type,
@@ -175,7 +173,7 @@ def process_sms_or_email_notification(
form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number']
send_to = validate_and_format_recipient(send_to=form_send_to,
key_type=api_key.key_type,
key_type=api_user.key_type,
service=service,
notification_type=notification_type)
@@ -190,8 +188,7 @@ def process_sms_or_email_notification(
if document_download_count:
# We changed personalisation which means we need to update the content
template_with_content.values = personalisation
key_type = api_key.key_type
service_in_research_mode = service.research_mode
resp = create_response_for_post_notification(
notification_id=notification_id,
client_reference=form.get('reference', None),
@@ -203,7 +200,7 @@ def process_sms_or_email_notification(
template_with_content=template_with_content
)
if str(service.id) in current_app.config.get('HIGH_VOLUME_SERVICE') and api_key.key_type == KEY_TYPE_NORMAL \
if service.id in current_app.config.get('HIGH_VOLUME_SERVICE') and api_user.key_type == KEY_TYPE_NORMAL \
and notification_type == EMAIL_TYPE:
# Put GOV.UK Email notifications onto a queue
# To take the pressure off the db for API requests put the notification for our high volume service onto a queue
@@ -214,7 +211,7 @@ def process_sms_or_email_notification(
form=form,
notification_id=str(notification_id),
notification_type=notification_type,
api_key=api_key,
api_key=api_user,
template=template,
service_id=service.id,
personalisation=personalisation,
@@ -237,8 +234,8 @@ def process_sms_or_email_notification(
service=service,
personalisation=personalisation,
notification_type=notification_type,
api_key_id=api_key.id,
key_type=key_type,
api_key_id=api_user.id,
key_type=api_user.key_type,
client_reference=form.get('reference', None),
simulated=simulated,
reply_to_text=reply_to_text,
@@ -248,10 +245,10 @@ def process_sms_or_email_notification(
if not simulated:
queue_name = QueueNames.PRIORITY if template_process_type == PRIORITY else None
send_notification_to_queue_detached(
key_type=key_type,
key_type=api_user.key_type,
notification_type=notification_type,
notification_id=notification_id,
research_mode=service_in_research_mode, # research_mode is deprecated
research_mode=service.research_mode, # research_mode is deprecated
queue=queue_name
)
else:

4
requirements-app.txt
View File

@@ -20,6 +20,8 @@ marshmallow==2.21.0 # pyup: <3 # v3 throws errors
psycopg2-binary==2.8.5
PyJWT==1.7.1
SQLAlchemy==1.3.17
strict-rfc3339==0.7
rfc3987==1.3.8
cachetools==4.1.0
notifications-python-client==5.5.1
@@ -31,4 +33,4 @@ git+https://github.com/alphagov/notifications-utils.git@40.2.1#egg=notifications
# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
prometheus-client==0.7.1
gds-metrics==0.2.0
gds-metrics==0.2.2

10
requirements.txt
View File

@@ -22,6 +22,8 @@ marshmallow==2.21.0 # pyup: <3 # v3 throws errors
psycopg2-binary==2.8.5
PyJWT==1.7.1
SQLAlchemy==1.3.17
strict-rfc3339==0.7
rfc3987==1.3.8
cachetools==4.1.0
notifications-python-client==5.5.1
@@ -33,21 +35,21 @@ git+https://github.com/alphagov/notifications-utils.git@40.2.1#egg=notifications
# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
prometheus-client==0.7.1
gds-metrics==0.2.0
gds-metrics==0.2.2
## The following requirements were added by pip freeze:
alembic==1.4.2
amqp==1.4.9
anyjson==0.3.3
attrs==19.3.0
awscli==1.18.93
awscli==1.18.97
bcrypt==3.1.7
billiard==3.3.0.23
bleach==3.1.4
blinker==1.4
boto==2.49.0
boto3==1.10.38
botocore==1.17.16
botocore==1.17.20
certifi==2020.6.20
chardet==3.0.4
click==7.1.2
@@ -80,7 +82,7 @@ pytz==2020.1
PyYAML==5.3.1
redis==3.5.3
requests==2.24.0
rsa==3.4.2
rsa==4.5
s3transfer==0.3.3
six==1.15.0
smartypants==2.0.1

3
requirements_for_test.txt
View File

@@ -8,8 +8,5 @@ pytest-cov==2.8.1
pytest-xdist==1.31.0
freezegun==0.3.12
requests-mock==1.7.0
# optional requirements for jsonschema
strict-rfc3339==0.7
rfc3987==1.3.8
# used for creating manifest file locally
jinja2-cli[yaml]==0.7.0

23
scripts/check_if_new_migration.py
View File

@@ -15,13 +15,24 @@ def get_latest_db_migration_to_apply():
def get_current_db_version():
api_status_url = '{}/_status'.format(os.getenv('API_HOST_NAME'))
response = requests.get(api_status_url)
if response.status_code != 200:
sys.exit('Could not make a request to the API: {}'.format())
current_db_version = response.json()['db_version']
return current_db_version
try:
response = requests.get(api_status_url)
response.raise_for_status()
current_db_version = response.json()['db_version']
return current_db_version
except requests.exceptions.ConnectionError:
print(f'Could not make web request to {api_status_url}', file=sys.stderr)
return ''
except Exception: # we expect these to be either either a http status code error, or a json decoding error
print(
f'Could not read status endpoint!\n\ncode {response.status_code}\nresponse "{response.text}"',
file=sys.stderr
)
# if we can't make a request to the API, the API is probably down. By returning a blank string (which won't
# match the filename of the latest migration), we force the migration to run, as the code change to fix the api
# might involve a migration file.
return ''
def run():

0
tests/app/broadcast_message/__init__.py Normal file
View File

282
tests/app/broadcast_message/test_rest.py Normal file
View File

@@ -0,0 +1,282 @@
import uuid
from freezegun import freeze_time
import pytest
from app.models import BROADCAST_TYPE, BroadcastStatusType
from tests.app.db import create_broadcast_message, create_template, create_service, create_user
def test_get_broadcast_message(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, areas=['place A', 'region B'])
response = admin_request.get(
'broadcast_message.get_broadcast_message',
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['id'] == str(bm.id)
assert response['template_name'] == t.name
assert response['status'] == BroadcastStatusType.DRAFT
assert response['created_at'] is not None
assert response['starts_at'] is None
assert response['areas'] == ['place A', 'region B']
assert response['personalisation'] == {}
def test_get_broadcast_message_404s_if_message_doesnt_exist(admin_request, sample_service):
err = admin_request.get(
'broadcast_message.get_broadcast_message',
service_id=sample_service.id,
broadcast_message_id=uuid.uuid4(),
_expected_status=404
)
assert err == {'message': 'No result found', 'result': 'error'}
def test_get_broadcast_message_404s_if_message_is_for_different_service(admin_request, sample_service):
other_service = create_service(service_name='other')
other_template = create_template(other_service, BROADCAST_TYPE)
bm = create_broadcast_message(other_template)
err = admin_request.get(
'broadcast_message.get_broadcast_message',
service_id=sample_service.id,
broadcast_message_id=bm.id,
_expected_status=404
)
assert err == {'message': 'No result found', 'result': 'error'}
@freeze_time('2020-01-01')
def test_get_broadcast_messages_for_service(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
with freeze_time('2020-01-01 12:00'):
bm1 = create_broadcast_message(t, personalisation={'foo': 'bar'})
with freeze_time('2020-01-01 13:00'):
bm2 = create_broadcast_message(t, personalisation={'foo': 'baz'})
response = admin_request.get(
'broadcast_message.get_broadcast_messages_for_service',
service_id=t.service_id,
_expected_status=200
)
assert response['broadcast_messages'][0]['id'] == str(bm1.id)
assert response['broadcast_messages'][1]['id'] == str(bm2.id)
@freeze_time('2020-01-01')
def test_create_broadcast_message(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
response = admin_request.post(
'broadcast_message.create_broadcast_message',
_data={
'template_id': str(t.id),
'service_id': str(t.service_id),
'created_by': str(t.created_by_id),
},
service_id=t.service_id,
_expected_status=201
)
assert response['template_name'] == t.name
assert response['status'] == BroadcastStatusType.DRAFT
assert response['created_at'] is not None
assert response['created_by_id'] == str(t.created_by_id)
assert response['personalisation'] == {}
assert response['areas'] == []
@pytest.mark.parametrize('data, expected_errors', [
(
{},
[
{'error': 'ValidationError', 'message': 'template_id is a required property'},
{'error': 'ValidationError', 'message': 'service_id is a required property'},
{'error': 'ValidationError', 'message': 'created_by is a required property'}
]
),
(
{
'template_id': str(uuid.uuid4()),
'service_id': str(uuid.uuid4()),
'created_by': str(uuid.uuid4()),
'foo': 'something else'
},
[
{'error': 'ValidationError', 'message': 'Additional properties are not allowed (foo was unexpected)'}
]
)
])
def test_create_broadcast_message_400s_if_json_schema_fails_validation(
admin_request,
sample_service,
data,
expected_errors
):
t = create_template(sample_service, BROADCAST_TYPE)
response = admin_request.post(
'broadcast_message.create_broadcast_message',
_data=data,
service_id=t.service_id,
_expected_status=400
)
assert response['errors'] == expected_errors
def test_update_broadcast_message(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, areas=['manchester'])
response = admin_request.post(
'broadcast_message.update_broadcast_message',
_data={'starts_at': '2020-06-01 20:00:01', 'areas': ['london', 'glasgow']},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['starts_at'] == '2020-06-01T20:00:01.000000Z'
assert response['areas'] == ['london', 'glasgow']
assert response['updated_at'] is not None
def test_update_broadcast_message_sets_finishes_at_separately(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, areas=['manchester'])
response = admin_request.post(
'broadcast_message.update_broadcast_message',
_data={'starts_at': '2020-06-01 20:00:01', 'finishes_at': '2020-06-02 20:00:01'},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['starts_at'] == '2020-06-01T20:00:01.000000Z'
assert response['finishes_at'] == '2020-06-02T20:00:01.000000Z'
assert response['updated_at'] is not None
@pytest.mark.parametrize('input_dt', [
'2020-06-01 20:00:01',
'2020-06-01T20:00:01',
'2020-06-01 20:00:01Z',
'2020-06-01T20:00:01+00:00',
])
def test_update_broadcast_message_allows_sensible_datetime_formats(admin_request, sample_service, input_dt):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t)
response = admin_request.post(
'broadcast_message.update_broadcast_message',
_data={'starts_at': input_dt},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['starts_at'] == '2020-06-01T20:00:01.000000Z'
assert response['updated_at'] is not None
def test_update_broadcast_message_doesnt_let_you_update_status(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t)
response = admin_request.post(
'broadcast_message.update_broadcast_message',
_data={'areas': ['glasgow'], 'status': BroadcastStatusType.BROADCASTING},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=400
)
assert response['errors'] == [{
'error': 'ValidationError',
'message': 'Additional properties are not allowed (status was unexpected)'
}]
def test_update_broadcast_message_status(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, status=BroadcastStatusType.DRAFT)
response = admin_request.post(
'broadcast_message.update_broadcast_message_status',
_data={'status': BroadcastStatusType.PENDING_APPROVAL, 'created_by': str(t.created_by_id)},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['status'] == BroadcastStatusType.PENDING_APPROVAL
assert response['updated_at'] is not None
def test_update_broadcast_message_status_doesnt_let_you_update_other_things(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t)
response = admin_request.post(
'broadcast_message.update_broadcast_message_status',
_data={'areas': ['glasgow'], 'status': BroadcastStatusType.BROADCASTING, 'created_by': str(t.created_by_id)},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=400
)
assert response['errors'] == [{
'error': 'ValidationError',
'message': 'Additional properties are not allowed (areas was unexpected)'
}]
def test_update_broadcast_message_status_stores_cancelled_by_and_cancelled_at(admin_request, sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, status=BroadcastStatusType.BROADCASTING)
canceller = create_user('canceller@gov.uk')
response = admin_request.post(
'broadcast_message.update_broadcast_message_status',
_data={'status': BroadcastStatusType.CANCELLED, 'created_by': str(canceller.id)},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['status'] == BroadcastStatusType.CANCELLED
assert response['cancelled_at'] is not None
assert response['cancelled_by_id'] == str(canceller.id)
def test_update_broadcast_message_status_stores_approved_by_and_approved_at_and_queues_task(
admin_request,
sample_service,
mocker
):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, status=BroadcastStatusType.PENDING_APPROVAL)
approver = create_user('approver@gov.uk')
mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_message.apply_async')
response = admin_request.post(
'broadcast_message.update_broadcast_message_status',
_data={'status': BroadcastStatusType.BROADCASTING, 'created_by': str(approver.id)},
service_id=t.service_id,
broadcast_message_id=bm.id,
_expected_status=200
)
assert response['status'] == BroadcastStatusType.BROADCASTING
assert response['approved_at'] is not None
assert response['approved_by_id'] == str(approver.id)
mock_task.assert_called_once_with(kwargs={'broadcast_message_id': str(bm.id)}, queue='notify-internal-tasks')

61
tests/app/celery/test_broadcast_message_tasks.py Normal file
View File

@@ -0,0 +1,61 @@
import pytest
import requests_mock
from requests import RequestException
from app.dao.templates_dao import dao_update_template
from app.models import BROADCAST_TYPE, BroadcastStatusType
from app.celery.broadcast_message_tasks import send_broadcast_message
from tests.app.db import create_template, create_broadcast_message
def test_send_broadcast_message_sends_data_correctly(sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, areas=['london'], status=BroadcastStatusType.BROADCASTING)
with requests_mock.Mocker() as request_mock:
request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", json={'valid': 'true'}, status_code=200)
send_broadcast_message(broadcast_message_id=str(bm.id))
assert request_mock.call_count == 1
assert request_mock.request_history[0].method == 'POST'
assert request_mock.request_history[0].headers["Content-type"] == "application/json"
cbc_json = request_mock.request_history[0].json()
assert cbc_json['template']['id'] == str(t.id)
assert cbc_json['template']['template_type'] == BROADCAST_TYPE
assert cbc_json['broadcast_message']['areas'] == ['london']
def test_send_broadcast_message_sends_old_version_of_template(sample_service):
t = create_template(sample_service, BROADCAST_TYPE, content='first content')
bm = create_broadcast_message(t, areas=['london'], status=BroadcastStatusType.BROADCASTING)
t.content = 'second content'
dao_update_template(t)
assert t.version == 2
with requests_mock.Mocker() as request_mock:
request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", json={'valid': 'true'}, status_code=200)
send_broadcast_message(broadcast_message_id=str(bm.id))
assert request_mock.call_count == 1
assert request_mock.request_history[0].method == 'POST'
assert request_mock.request_history[0].headers["Content-type"] == "application/json"
cbc_json = request_mock.request_history[0].json()
assert cbc_json['template']['id'] == str(t.id)
assert cbc_json['template']['version'] == 1
assert cbc_json['template']['content'] == 'first content'
def test_send_broadcast_message_errors(sample_service):
t = create_template(sample_service, BROADCAST_TYPE)
bm = create_broadcast_message(t, status=BroadcastStatusType.BROADCASTING)
with requests_mock.Mocker() as request_mock:
request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", text='503 bad gateway', status_code=503)
# we're not retrying or anything for the moment - but this'll ensure any exception gets logged
with pytest.raises(RequestException) as ex:
send_broadcast_message(broadcast_message_id=str(bm.id))
assert ex.value.response.status_code == 503

82
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -155,20 +155,19 @@ def test_update_billable_units_for_letter_doesnt_update_if_sent_with_test_key(mo
@freeze_time('2020-02-17 18:00:00')
def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sample_letter_template):
# second class
create_notification(
template=sample_letter_template,
status='created',
reference='ref0',
created_at=(datetime.now() - timedelta(hours=2))
)
create_notification(
template=sample_letter_template,
status='created',
reference='ref1',
created_at=(datetime.now() - timedelta(hours=3))
)
create_notification(
template=sample_letter_template,
status='created',
@@ -208,7 +207,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
{'ContentLength': 3},
])
results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second')
assert mock_s3.call_count == 3
mock_s3.assert_has_calls(
@@ -220,6 +219,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
)
assert len(results) == 3
assert results == [
{'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2},
{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1},
@@ -256,7 +256,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
ClientError(error_response, "File not found")
])
results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second')
assert mock_head_s3_object.call_count == 2
mock_head_s3_object.assert_has_calls(
@@ -275,20 +275,19 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
])
def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mocker, time_to_run_task):
with freeze_time("2020-02-17 18:00:00"):
# second class
create_notification(
template=sample_letter_template,
status='created',
reference='ref0',
created_at=(datetime.now() - timedelta(hours=2))
)
create_notification(
template=sample_letter_template,
status='created',
reference='ref1',
created_at=(datetime.now() - timedelta(hours=3))
)
create_notification(
template=sample_letter_template,
status='created',
@@ -296,10 +295,38 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
created_at=(datetime.now() - timedelta(days=2))
)
# first class
create_notification(
template=sample_letter_template,
status='created',
reference='first_class',
created_at=(datetime.now() - timedelta(hours=4)),
postage="first"
)
# international
create_notification(
template=sample_letter_template,
status='created',
reference='international',
created_at=(datetime.now() - timedelta(days=3)),
postage="europe"
)
create_notification(
template=sample_letter_template,
status='created',
reference='international',
created_at=(datetime.now() - timedelta(days=4)),
postage="rest-of-world"
)
mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[
{'ContentLength': 1},
{'ContentLength': 1},
{'ContentLength': 2},
{'ContentLength': 1},
{'ContentLength': 3},
{'ContentLength': 1},
])
mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task')
@@ -308,26 +335,59 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
with freeze_time(time_to_run_task):
collate_letter_pdfs_to_be_sent()
assert len(mock_celery.call_args_list) == 2
assert len(mock_celery.call_args_list) == 5
assert mock_celery.call_args_list[0] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
'2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'
'2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'
],
'upload_filename': 'NOTIFY.2020-02-17.001.k3x_WqC5KhB6e2DWv9Ma.ZIP'
'upload_filename': 'NOTIFY.2020-02-17.1.001.kHh01fdUxT9iEIYUt5Wx.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[1] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
'2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'
],
'upload_filename': 'NOTIFY.2020-02-17.2.001.k3x_WqC5KhB6e2DWv9Ma.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[2] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'
],
'upload_filename': 'NOTIFY.2020-02-17.002.J85cUw-FWlKuAIOcwdLS.ZIP'
'upload_filename': 'NOTIFY.2020-02-17.2.002.J85cUw-FWlKuAIOcwdLS.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[3] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'
],
'upload_filename': 'NOTIFY.2020-02-17.E.001.4YajCZzgzIl7zf8bjWK2.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[4] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF',
],
'upload_filename': 'NOTIFY.2020-02-17.N.001.eSvP8Ph6EBKhh3k7BSA2.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'

29
tests/app/db.py
View File

@@ -59,7 +59,9 @@ from app.models import (
Domain,
NotificationHistory,
ReturnedLetter,
ServiceContactList
ServiceContactList,
BroadcastMessage,
BroadcastStatusType,
)
@@ -984,3 +986,28 @@ def create_service_contact_list(
db.session.add(contact_list)
db.session.commit()
return contact_list
def create_broadcast_message(
template,
created_by=None,
personalisation={},
status=BroadcastStatusType.DRAFT,
starts_at=None,
finishes_at=None,
areas=[],
):
broadcast_message = BroadcastMessage(
service_id=template.service_id,
template_id=template.id,
template_version=template.version,
personalisation=personalisation,
status=BroadcastStatusType.DRAFT,
starts_at=starts_at,
finishes_at=finishes_at,
created_by_id=created_by.id if created_by else template.created_by_id,
areas=areas,
)
db.session.add(broadcast_message)
db.session.commit()
return broadcast_message