diff --git a/app/__init__.py b/app/__init__.py index 52a527021..79c57e9ef 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -78,12 +78,14 @@ def create_app(application): from app.config import configs notify_environment = os.environ['NOTIFY_ENVIRONMENT'] - print(notify_environment) application.config.from_object(configs[notify_environment]) application.config['NOTIFY_APP_NAME'] = application.name init_app(application) + + # Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded + metrics.init_app(application) request_helper.init_app(application) db.init_app(application) migrate.init_app(application, db=db) @@ -109,7 +111,6 @@ def create_app(application): redis_store.init_app(application) performance_platform_client.init_app(application) document_download_client.init_app(application) - metrics.init_app(application) register_blueprint(application) register_v2_blueprints(application) @@ -153,6 +154,7 @@ def register_blueprint(application): from app.template_folder.rest import template_folder_blueprint from app.letter_branding.letter_branding_rest import letter_branding_blueprint from app.upload.rest import upload_blueprint + from app.broadcast_message.rest import broadcast_message_blueprint service_blueprint.before_request(requires_admin_auth) application.register_blueprint(service_blueprint, url_prefix='/service') @@ -238,6 +240,9 @@ def register_blueprint(application): upload_blueprint.before_request(requires_admin_auth) application.register_blueprint(upload_blueprint) + broadcast_message_blueprint.before_request(requires_admin_auth) + application.register_blueprint(broadcast_message_blueprint) + def register_v2_blueprints(application): from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms diff --git a/app/broadcast_message/__init__.py b/app/broadcast_message/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/broadcast_message/broadcast_message_schema.py b/app/broadcast_message/broadcast_message_schema.py new file mode 100644 index 000000000..743ca54d7 --- /dev/null +++ b/app/broadcast_message/broadcast_message_schema.py @@ -0,0 +1,48 @@ +from app.schema_validation.definitions import uuid +from app.models import BroadcastStatusType + +create_broadcast_message_schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'description': 'POST create broadcast_message schema', + 'type': 'object', + 'title': 'Create broadcast_message', + 'properties': { + 'template_id': uuid, + 'service_id': uuid, + 'created_by': uuid, + 'personalisation': {'type': 'object'}, + 'starts_at': {'type': 'string', 'format': 'datetime'}, + 'finishes_at': {'type': 'string', 'format': 'datetime'}, + 'areas': {"type": "array", "items": {"type": "string"}}, + }, + 'required': ['template_id', 'service_id', 'created_by'], + 'additionalProperties': False +} + +update_broadcast_message_schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'description': 'POST update broadcast_message schema', + 'type': 'object', + 'title': 'Update broadcast_message', + 'properties': { + 'personalisation': {'type': 'object'}, + 'starts_at': {'type': 'string', 'format': 'datetime'}, + 'finishes_at': {'type': 'string', 'format': 'datetime'}, + 'areas': {"type": "array", "items": {"type": "string"}}, + }, + 'required': [], + 'additionalProperties': False +} + +update_broadcast_message_status_schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'description': 'POST update broadcast_message status schema', + 'type': 'object', + 'title': 'Update broadcast_message', + 'properties': { + 'status': {'type': 'string', 'enum': BroadcastStatusType.STATUSES}, + 'created_by': uuid, + }, + 'required': ['status', 'created_by'], + 'additionalProperties': False +} diff --git a/app/broadcast_message/rest.py b/app/broadcast_message/rest.py new file mode 100644 index 000000000..e75a19079 --- /dev/null +++ b/app/broadcast_message/rest.py @@ -0,0 +1,133 @@ +from datetime import datetime + +import iso8601 +from flask import Blueprint, jsonify, request, current_app + +from app.config import QueueNames +from app.dao.templates_dao import dao_get_template_by_id_and_service_id +from app.dao.users_dao import get_user_by_id +from app.dao.broadcast_message_dao import ( + dao_create_broadcast_message, + dao_get_broadcast_message_by_id_and_service_id, + dao_get_broadcast_messages_for_service, + dao_update_broadcast_message, +) +from app.dao.services_dao import dao_fetch_service_by_id +from app.errors import register_errors +from app.models import BroadcastMessage, BroadcastStatusType +from app.celery.broadcast_message_tasks import send_broadcast_message +from app.broadcast_message.broadcast_message_schema import ( + create_broadcast_message_schema, + update_broadcast_message_schema, + update_broadcast_message_status_schema, +) +from app.schema_validation import validate + +broadcast_message_blueprint = Blueprint( + 'broadcast_message', + __name__, + url_prefix='/service//broadcast-message' +) +register_errors(broadcast_message_blueprint) + + +def _parse_nullable_datetime(dt): + if dt: + return iso8601.parse_date(dt).replace(tzinfo=None) + return dt + + +@broadcast_message_blueprint.route('', methods=['GET']) +def get_broadcast_messages_for_service(service_id): + # TODO: should this return template content/data in some way? or can we rely on them being cached admin side. + # we might need stuff like template name for showing on the dashboard. + # TODO: should this paginate or filter on dates or anything? + broadcast_messages = [o.serialize() for o in dao_get_broadcast_messages_for_service(service_id)] + return jsonify(broadcast_messages=broadcast_messages) + + +@broadcast_message_blueprint.route('/', methods=['GET']) +def get_broadcast_message(service_id, broadcast_message_id): + return jsonify(dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id).serialize()) + + +@broadcast_message_blueprint.route('', methods=['POST']) +def create_broadcast_message(service_id): + data = request.get_json() + + validate(data, create_broadcast_message_schema) + service = dao_fetch_service_by_id(data['service_id']) + user = get_user_by_id(data['created_by']) + template = dao_get_template_by_id_and_service_id(data['template_id'], data['service_id']) + + broadcast_message = BroadcastMessage( + service_id=service.id, + template_id=template.id, + template_version=template.version, + personalisation=data.get('personalisation', {}), + areas=data.get('areas', []), + status=BroadcastStatusType.DRAFT, + starts_at=_parse_nullable_datetime(data.get('starts_at')), + finishes_at=_parse_nullable_datetime(data.get('finishes_at')), + created_by_id=user.id, + ) + + dao_create_broadcast_message(broadcast_message) + + return jsonify(broadcast_message.serialize()), 201 + + +@broadcast_message_blueprint.route('/', methods=['POST']) +def update_broadcast_message(service_id, broadcast_message_id): + data = request.get_json() + + validate(data, update_broadcast_message_schema) + + broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id) + + if 'personalisation' in data: + broadcast_message.personalisation = data['personalisation'] + if 'starts_at' in data: + broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at']) + if 'finishes_at' in data: + broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at']) + if 'areas' in data: + broadcast_message.areas = data['areas'] + + dao_update_broadcast_message(broadcast_message) + + return jsonify(broadcast_message.serialize()), 200 + + +@broadcast_message_blueprint.route('//status', methods=['POST']) +def update_broadcast_message_status(service_id, broadcast_message_id): + data = request.get_json() + + validate(data, update_broadcast_message_status_schema) + broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id) + + new_status = data['status'] + + # TODO: Restrict status transitions + # TODO: validate that the user belongs to the same service, isn't the creator, has permissions, etc + if new_status == BroadcastStatusType.BROADCASTING: + broadcast_message.approved_at = datetime.utcnow() + broadcast_message.approved_by = get_user_by_id(data['created_by']) + if new_status == BroadcastStatusType.CANCELLED: + broadcast_message.cancelled_at = datetime.utcnow() + broadcast_message.cancelled_by = get_user_by_id(data['created_by']) + + broadcast_message.status = new_status + + current_app.logger.info( + f'broadcast_message {broadcast_message_id} moving from {broadcast_message.status} to {new_status}' + ) + dao_update_broadcast_message(broadcast_message) + + if new_status == BroadcastStatusType.BROADCASTING: + send_broadcast_message.apply_async( + kwargs={'broadcast_message_id': str(broadcast_message.id)}, + queue=QueueNames.NOTIFY + ) + + return jsonify(broadcast_message.serialize()), 200 diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py new file mode 100644 index 000000000..837bf352b --- /dev/null +++ b/app/celery/broadcast_message_tasks.py @@ -0,0 +1,37 @@ +import requests +from flask import current_app +from notifications_utils.statsd_decorators import statsd + +from app import notify_celery + +from app.dao.broadcast_message_dao import dao_get_broadcast_message_by_id + + +@notify_celery.task(name="send-broadcast-message") +@statsd(namespace="tasks") +def send_broadcast_message(broadcast_message_id, provider='stub-1'): + # imports of schemas from tasks have to happen within functions to prevent + # `AttributeError: 'DummySession' object has no attribute 'query'` errors in unrelated tests + from app.schemas import template_schema + + broadcast_message = dao_get_broadcast_message_by_id(broadcast_message_id) + + current_app.logger.info( + f'sending broadcast_message {broadcast_message_id} ' + f'status {broadcast_message.status} to {provider}' + ) + + payload = { + "template": template_schema.dump(broadcast_message.template).data, + "broadcast_message": broadcast_message.serialize(), + } + resp = requests.post( + f'{current_app.config["CBC_PROXY_URL"]}/broadcasts/{provider}', + json=payload + ) + resp.raise_for_status() + + current_app.logger.info( + f'broadcast_message {broadcast_message.id} ' + f'status {broadcast_message.status} sent to {provider}' + ) diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 8c1ce0e3c..61982e0b4 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -42,6 +42,8 @@ from app.models import ( NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_VALIDATION_FAILED, NOTIFICATION_VIRUS_SCAN_FAILED, + POSTAGE_TYPES, + RESOLVE_POSTAGE_FOR_FILE_NAME ) from app.cronitor import cronitor @@ -127,6 +129,7 @@ def collate_letter_pdfs_to_be_sent(): that have not yet been sent. If run after midnight, it will collect up letters created before 5:30pm the day before. """ + current_app.logger.info("starting collate-letter-pdfs-to-be-sent") print_run_date = convert_utc_to_bst(datetime.utcnow()) if print_run_date.time() < LETTER_PROCESSING_DEADLINE: print_run_date = print_run_date - timedelta(days=1) @@ -134,41 +137,45 @@ def collate_letter_pdfs_to_be_sent(): print_run_deadline = print_run_date.replace( hour=17, minute=30, second=0, microsecond=0 ) + for postage in POSTAGE_TYPES: + current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}") + letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage) - letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) + for i, letters in enumerate(group_letters(letters_to_print)): + filenames = [letter['Key'] for letter in letters] - for i, letters in enumerate(group_letters(letters_to_print)): - filenames = [letter['Key'] for letter in letters] - - hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() - # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP - dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( - date=print_run_deadline.strftime("%Y-%m-%d"), - num=i + 1, - hash=hash - ) - - current_app.logger.info( - 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( - len(filenames), - dvla_filename, - sum(letter['Size'] for letter in letters) + hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() + # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP + dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format( + date=print_run_deadline.strftime("%Y-%m-%d"), + postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage], + num=i + 1, + hash=hash ) - ) - notify_celery.send_task( - name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, - kwargs={ - 'filenames_to_zip': filenames, - 'upload_filename': dvla_filename - }, - queue=QueueNames.PROCESS_FTP, - compression='zlib' - ) + + current_app.logger.info( + 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( + len(filenames), + dvla_filename, + sum(letter['Size'] for letter in letters) + ) + ) + notify_celery.send_task( + name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, + kwargs={ + 'filenames_to_zip': filenames, + 'upload_filename': dvla_filename + }, + queue=QueueNames.PROCESS_FTP, + compression='zlib' + ) + current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}") + + current_app.logger.info("finished collate-letter-pdfs-to-be-sent") -def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): - letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) - +def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage): + letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage) letter_pdfs = [] for letter in letters_awaiting_sending: try: diff --git a/app/config.py b/app/config.py index 0ada716b0..b0e3b555c 100644 --- a/app/config.py +++ b/app/config.py @@ -112,6 +112,9 @@ class Config(object): # Antivirus ANTIVIRUS_ENABLED = True + # Broadcast Messaging + CBC_PROXY_URL = None + ########################### # Default config values ### ########################### @@ -393,6 +396,8 @@ class Development(Config): API_HOST_NAME = "http://localhost:6011" API_RATE_LIMIT_ENABLED = True + CBC_PROXY_URL = 'http://localhost:8080' + class Test(Development): NOTIFY_EMAIL_DOMAIN = 'test.notify.com' @@ -436,6 +441,8 @@ class Test(Development): FIRETEXT_INBOUND_SMS_AUTH = ['testkey'] TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999' + CBC_PROXY_URL = 'http://test-cbc-proxy' + MMG_URL = 'https://example.com/mmg' FIRETEXT_URL = 'https://example.com/firetext' @@ -452,6 +459,7 @@ class Preview(Config): INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf' TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise' + CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital' FROM_NUMBER = 'preview' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = False @@ -469,6 +477,7 @@ class Staging(Config): INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf' TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise' + CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital' FROM_NUMBER = 'stage' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True @@ -487,6 +496,7 @@ class Live(Config): INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' + CBC_PROXY_URL = 'https://notify-stub-cbc-sandbox.cloudapps.digital' FROM_NUMBER = 'GOVUK' PERFORMANCE_PLATFORM_ENABLED = True API_RATE_LIMIT_ENABLED = True diff --git a/app/dao/broadcast_message_dao.py b/app/dao/broadcast_message_dao.py new file mode 100644 index 000000000..8e5849394 --- /dev/null +++ b/app/dao/broadcast_message_dao.py @@ -0,0 +1,30 @@ +from app import db +from app.models import BroadcastMessage +from app.dao.dao_utils import transactional + + +@transactional +def dao_create_broadcast_message(broadcast_message): + db.session.add(broadcast_message) + + +@transactional +def dao_update_broadcast_message(broadcast_message): + db.session.add(broadcast_message) + + +def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id): + return BroadcastMessage.query.filter( + BroadcastMessage.id == broadcast_message_id, + BroadcastMessage.service_id == service_id + ).one() + + +def dao_get_broadcast_message_by_id(broadcast_message_id): + return BroadcastMessage.query.get(broadcast_message_id) + + +def dao_get_broadcast_messages_for_service(service_id): + return BroadcastMessage.query.filter( + BroadcastMessage.service_id == service_id + ).order_by(BroadcastMessage.created_at) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 8bc62657e..6aa976b20 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -733,7 +733,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ return notifications -def dao_get_letters_to_be_printed(print_run_deadline): +def dao_get_letters_to_be_printed(print_run_deadline, postage): """ Return all letters created before the print run deadline that have not yet been sent """ @@ -741,7 +741,8 @@ def dao_get_letters_to_be_printed(print_run_deadline): Notification.created_at < convert_bst_to_utc(print_run_deadline), Notification.notification_type == LETTER_TYPE, Notification.status == NOTIFICATION_CREATED, - Notification.key_type == KEY_TYPE_NORMAL + Notification.key_type == KEY_TYPE_NORMAL, + Notification.postage == postage, ).order_by( Notification.created_at ).all() diff --git a/app/models.py b/app/models.py index eae9c921a..81670144c 100644 --- a/app/models.py +++ b/app/models.py @@ -2174,7 +2174,7 @@ class BroadcastMessage(db.Model): {} ) - id = db.Column(UUID(as_uuid=True), primary_key=True) + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id')) service = db.relationship('Service', backref='broadcast_messages') @@ -2184,6 +2184,8 @@ class BroadcastMessage(db.Model): template = db.relationship('TemplateHistory', backref='broadcast_messages') _personalisation = db.Column(db.String, nullable=True) + # defaults to empty list + areas = db.Column(JSONB(none_as_null=True), nullable=False, default=list) status = db.Column( db.String, @@ -2197,7 +2199,7 @@ class BroadcastMessage(db.Model): finishes_at = db.Column(db.DateTime, nullable=True) # isn't updated if user cancels # these times correspond to when - created_at = db.Column(db.DateTime, nullable=False) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) approved_at = db.Column(db.DateTime, nullable=True) cancelled_at = db.Column(db.DateTime, nullable=True) updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) @@ -2209,3 +2211,41 @@ class BroadcastMessage(db.Model): created_by = db.relationship('User', foreign_keys=[created_by_id]) approved_by = db.relationship('User', foreign_keys=[approved_by_id]) cancelled_by = db.relationship('User', foreign_keys=[cancelled_by_id]) + + @property + def personalisation(self): + if self._personalisation: + return encryption.decrypt(self._personalisation) + return {} + + @personalisation.setter + def personalisation(self, personalisation): + self._personalisation = encryption.encrypt(personalisation or {}) + + def serialize(self): + return { + 'id': str(self.id), + + 'service_id': str(self.service_id), + + 'template_id': str(self.template_id), + 'template_version': self.template_version, + 'template_name': self.template.name, + + 'personalisation': self.personalisation, + 'areas': self.areas, + + 'status': self.status, + + 'starts_at': self.starts_at.strftime(DATETIME_FORMAT) if self.starts_at else None, + 'finishes_at': self.finishes_at.strftime(DATETIME_FORMAT) if self.finishes_at else None, + + 'created_at': self.created_at.strftime(DATETIME_FORMAT) if self.created_at else None, + 'approved_at': self.approved_at.strftime(DATETIME_FORMAT) if self.approved_at else None, + 'cancelled_at': self.cancelled_at.strftime(DATETIME_FORMAT) if self.cancelled_at else None, + 'updated_at': self.updated_at.strftime(DATETIME_FORMAT) if self.updated_at else None, + + 'created_by_id': str(self.created_by_id), + 'approved_by_id': str(self.approved_by_id), + 'cancelled_by_id': str(self.cancelled_by_id), + } diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index e5ca0adcc..f6b14df18 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -150,17 +150,13 @@ def persist_notification( notification.postage = postage or template_postage notification.normalised_to = ''.join(notification.to.split()).lower() - # Get service attributes before the commit - service_in_trial_mode = service.restricted - service_id = service.id - # if simulated create a Notification model to return but do not persist the Notification to the dB if not simulated: dao_create_notification(notification) # Only keep track of the daily limit for trial mode services. - if service_in_trial_mode and key_type != KEY_TYPE_TEST: - if redis_store.get(redis.daily_limit_cache_key(service_id)): - redis_store.incr(redis.daily_limit_cache_key(service_id)) + if service.restricted and key_type != KEY_TYPE_TEST: + if redis_store.get(redis.daily_limit_cache_key(service.id)): + redis_store.incr(redis.daily_limit_cache_key(service.id)) current_app.logger.info( "{} {} created at {}".format(notification_type, notification_id, notification_created_at) diff --git a/app/schema_validation/__init__.py b/app/schema_validation/__init__.py index 0ddd51f16..f5f2dc9ef 100644 --- a/app/schema_validation/__init__.py +++ b/app/schema_validation/__init__.py @@ -54,6 +54,17 @@ def validate_schema_date_with_hour(instance): return True +@format_checker.checks('datetime', raises=ValidationError) +def validate_schema_datetime(instance): + if isinstance(instance, str): + try: + iso8601.parse_date(instance) + except ParseError: + raise ValidationError("datetime format is invalid. It must be a valid ISO8601 date time format, " + "https://en.wikipedia.org/wiki/ISO_8601") + return True + + def validate(json_to_validate, schema): validator = Draft7Validator(schema, format_checker=format_checker) errors = list(validator.iter_errors(json_to_validate)) diff --git a/app/serialised_models.py b/app/serialised_models.py index 50adeceeb..1b17726f0 100644 --- a/app/serialised_models.py +++ b/app/serialised_models.py @@ -47,7 +47,6 @@ class SerialisedTemplate(SerialisedModel): 'subject', 'template_type', 'version', - 'broadcast_data', } @classmethod diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index af0d766ea..aa204721a 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -149,7 +149,6 @@ def post_notification(notification_type): notification = process_sms_or_email_notification( form=form, notification_type=notification_type, - api_key=api_user, template=template, template_with_content=template_with_content, template_process_type=template.process_type, @@ -164,7 +163,6 @@ def process_sms_or_email_notification( *, form, notification_type, - api_key, template, template_with_content, template_process_type, @@ -175,7 +173,7 @@ def process_sms_or_email_notification( form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number'] send_to = validate_and_format_recipient(send_to=form_send_to, - key_type=api_key.key_type, + key_type=api_user.key_type, service=service, notification_type=notification_type) @@ -190,8 +188,7 @@ def process_sms_or_email_notification( if document_download_count: # We changed personalisation which means we need to update the content template_with_content.values = personalisation - key_type = api_key.key_type - service_in_research_mode = service.research_mode + resp = create_response_for_post_notification( notification_id=notification_id, client_reference=form.get('reference', None), @@ -203,7 +200,7 @@ def process_sms_or_email_notification( template_with_content=template_with_content ) - if str(service.id) in current_app.config.get('HIGH_VOLUME_SERVICE') and api_key.key_type == KEY_TYPE_NORMAL \ + if service.id in current_app.config.get('HIGH_VOLUME_SERVICE') and api_user.key_type == KEY_TYPE_NORMAL \ and notification_type == EMAIL_TYPE: # Put GOV.UK Email notifications onto a queue # To take the pressure off the db for API requests put the notification for our high volume service onto a queue @@ -214,7 +211,7 @@ def process_sms_or_email_notification( form=form, notification_id=str(notification_id), notification_type=notification_type, - api_key=api_key, + api_key=api_user, template=template, service_id=service.id, personalisation=personalisation, @@ -237,8 +234,8 @@ def process_sms_or_email_notification( service=service, personalisation=personalisation, notification_type=notification_type, - api_key_id=api_key.id, - key_type=key_type, + api_key_id=api_user.id, + key_type=api_user.key_type, client_reference=form.get('reference', None), simulated=simulated, reply_to_text=reply_to_text, @@ -248,10 +245,10 @@ def process_sms_or_email_notification( if not simulated: queue_name = QueueNames.PRIORITY if template_process_type == PRIORITY else None send_notification_to_queue_detached( - key_type=key_type, + key_type=api_user.key_type, notification_type=notification_type, notification_id=notification_id, - research_mode=service_in_research_mode, # research_mode is deprecated + research_mode=service.research_mode, # research_mode is deprecated queue=queue_name ) else: diff --git a/requirements-app.txt b/requirements-app.txt index 50f3b6f28..92703f439 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -20,6 +20,8 @@ marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.5 PyJWT==1.7.1 SQLAlchemy==1.3.17 +strict-rfc3339==0.7 +rfc3987==1.3.8 cachetools==4.1.0 notifications-python-client==5.5.1 @@ -31,4 +33,4 @@ git+https://github.com/alphagov/notifications-utils.git@40.2.1#egg=notifications # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.7.1 -gds-metrics==0.2.0 +gds-metrics==0.2.2 diff --git a/requirements.txt b/requirements.txt index e42c6931c..2421420ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,8 @@ marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.5 PyJWT==1.7.1 SQLAlchemy==1.3.17 +strict-rfc3339==0.7 +rfc3987==1.3.8 cachetools==4.1.0 notifications-python-client==5.5.1 @@ -33,21 +35,21 @@ git+https://github.com/alphagov/notifications-utils.git@40.2.1#egg=notifications # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.7.1 -gds-metrics==0.2.0 +gds-metrics==0.2.2 ## The following requirements were added by pip freeze: alembic==1.4.2 amqp==1.4.9 anyjson==0.3.3 attrs==19.3.0 -awscli==1.18.93 +awscli==1.18.97 bcrypt==3.1.7 billiard==3.3.0.23 bleach==3.1.4 blinker==1.4 boto==2.49.0 boto3==1.10.38 -botocore==1.17.16 +botocore==1.17.20 certifi==2020.6.20 chardet==3.0.4 click==7.1.2 @@ -80,7 +82,7 @@ pytz==2020.1 PyYAML==5.3.1 redis==3.5.3 requests==2.24.0 -rsa==3.4.2 +rsa==4.5 s3transfer==0.3.3 six==1.15.0 smartypants==2.0.1 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index f8edc4dc1..9db32b421 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -8,8 +8,5 @@ pytest-cov==2.8.1 pytest-xdist==1.31.0 freezegun==0.3.12 requests-mock==1.7.0 -# optional requirements for jsonschema -strict-rfc3339==0.7 -rfc3987==1.3.8 # used for creating manifest file locally jinja2-cli[yaml]==0.7.0 diff --git a/scripts/check_if_new_migration.py b/scripts/check_if_new_migration.py index 2878086f6..b49f790a9 100644 --- a/scripts/check_if_new_migration.py +++ b/scripts/check_if_new_migration.py @@ -15,13 +15,24 @@ def get_latest_db_migration_to_apply(): def get_current_db_version(): api_status_url = '{}/_status'.format(os.getenv('API_HOST_NAME')) - response = requests.get(api_status_url) - if response.status_code != 200: - sys.exit('Could not make a request to the API: {}'.format()) - - current_db_version = response.json()['db_version'] - return current_db_version + try: + response = requests.get(api_status_url) + response.raise_for_status() + current_db_version = response.json()['db_version'] + return current_db_version + except requests.exceptions.ConnectionError: + print(f'Could not make web request to {api_status_url}', file=sys.stderr) + return '' + except Exception: # we expect these to be either either a http status code error, or a json decoding error + print( + f'Could not read status endpoint!\n\ncode {response.status_code}\nresponse "{response.text}"', + file=sys.stderr + ) + # if we can't make a request to the API, the API is probably down. By returning a blank string (which won't + # match the filename of the latest migration), we force the migration to run, as the code change to fix the api + # might involve a migration file. + return '' def run(): diff --git a/tests/app/broadcast_message/__init__.py b/tests/app/broadcast_message/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/app/broadcast_message/test_rest.py b/tests/app/broadcast_message/test_rest.py new file mode 100644 index 000000000..39627e9b1 --- /dev/null +++ b/tests/app/broadcast_message/test_rest.py @@ -0,0 +1,282 @@ +import uuid + +from freezegun import freeze_time +import pytest + +from app.models import BROADCAST_TYPE, BroadcastStatusType + +from tests.app.db import create_broadcast_message, create_template, create_service, create_user + + +def test_get_broadcast_message(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, areas=['place A', 'region B']) + + response = admin_request.get( + 'broadcast_message.get_broadcast_message', + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['id'] == str(bm.id) + assert response['template_name'] == t.name + assert response['status'] == BroadcastStatusType.DRAFT + assert response['created_at'] is not None + assert response['starts_at'] is None + assert response['areas'] == ['place A', 'region B'] + assert response['personalisation'] == {} + + +def test_get_broadcast_message_404s_if_message_doesnt_exist(admin_request, sample_service): + err = admin_request.get( + 'broadcast_message.get_broadcast_message', + service_id=sample_service.id, + broadcast_message_id=uuid.uuid4(), + _expected_status=404 + ) + assert err == {'message': 'No result found', 'result': 'error'} + + +def test_get_broadcast_message_404s_if_message_is_for_different_service(admin_request, sample_service): + other_service = create_service(service_name='other') + other_template = create_template(other_service, BROADCAST_TYPE) + bm = create_broadcast_message(other_template) + + err = admin_request.get( + 'broadcast_message.get_broadcast_message', + service_id=sample_service.id, + broadcast_message_id=bm.id, + _expected_status=404 + ) + assert err == {'message': 'No result found', 'result': 'error'} + + +@freeze_time('2020-01-01') +def test_get_broadcast_messages_for_service(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + + with freeze_time('2020-01-01 12:00'): + bm1 = create_broadcast_message(t, personalisation={'foo': 'bar'}) + with freeze_time('2020-01-01 13:00'): + bm2 = create_broadcast_message(t, personalisation={'foo': 'baz'}) + + response = admin_request.get( + 'broadcast_message.get_broadcast_messages_for_service', + service_id=t.service_id, + _expected_status=200 + ) + + assert response['broadcast_messages'][0]['id'] == str(bm1.id) + assert response['broadcast_messages'][1]['id'] == str(bm2.id) + + +@freeze_time('2020-01-01') +def test_create_broadcast_message(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + + response = admin_request.post( + 'broadcast_message.create_broadcast_message', + _data={ + 'template_id': str(t.id), + 'service_id': str(t.service_id), + 'created_by': str(t.created_by_id), + }, + service_id=t.service_id, + _expected_status=201 + ) + + assert response['template_name'] == t.name + assert response['status'] == BroadcastStatusType.DRAFT + assert response['created_at'] is not None + assert response['created_by_id'] == str(t.created_by_id) + assert response['personalisation'] == {} + assert response['areas'] == [] + + +@pytest.mark.parametrize('data, expected_errors', [ + ( + {}, + [ + {'error': 'ValidationError', 'message': 'template_id is a required property'}, + {'error': 'ValidationError', 'message': 'service_id is a required property'}, + {'error': 'ValidationError', 'message': 'created_by is a required property'} + ] + ), + ( + { + 'template_id': str(uuid.uuid4()), + 'service_id': str(uuid.uuid4()), + 'created_by': str(uuid.uuid4()), + 'foo': 'something else' + }, + [ + {'error': 'ValidationError', 'message': 'Additional properties are not allowed (foo was unexpected)'} + ] + ) +]) +def test_create_broadcast_message_400s_if_json_schema_fails_validation( + admin_request, + sample_service, + data, + expected_errors +): + t = create_template(sample_service, BROADCAST_TYPE) + + response = admin_request.post( + 'broadcast_message.create_broadcast_message', + _data=data, + service_id=t.service_id, + _expected_status=400 + ) + assert response['errors'] == expected_errors + + +def test_update_broadcast_message(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, areas=['manchester']) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message', + _data={'starts_at': '2020-06-01 20:00:01', 'areas': ['london', 'glasgow']}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' + assert response['areas'] == ['london', 'glasgow'] + assert response['updated_at'] is not None + + +def test_update_broadcast_message_sets_finishes_at_separately(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, areas=['manchester']) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message', + _data={'starts_at': '2020-06-01 20:00:01', 'finishes_at': '2020-06-02 20:00:01'}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' + assert response['finishes_at'] == '2020-06-02T20:00:01.000000Z' + assert response['updated_at'] is not None + + +@pytest.mark.parametrize('input_dt', [ + '2020-06-01 20:00:01', + '2020-06-01T20:00:01', + '2020-06-01 20:00:01Z', + '2020-06-01T20:00:01+00:00', +]) +def test_update_broadcast_message_allows_sensible_datetime_formats(admin_request, sample_service, input_dt): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message', + _data={'starts_at': input_dt}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' + assert response['updated_at'] is not None + + +def test_update_broadcast_message_doesnt_let_you_update_status(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message', + _data={'areas': ['glasgow'], 'status': BroadcastStatusType.BROADCASTING}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=400 + ) + + assert response['errors'] == [{ + 'error': 'ValidationError', + 'message': 'Additional properties are not allowed (status was unexpected)' + }] + + +def test_update_broadcast_message_status(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, status=BroadcastStatusType.DRAFT) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message_status', + _data={'status': BroadcastStatusType.PENDING_APPROVAL, 'created_by': str(t.created_by_id)}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['status'] == BroadcastStatusType.PENDING_APPROVAL + assert response['updated_at'] is not None + + +def test_update_broadcast_message_status_doesnt_let_you_update_other_things(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t) + + response = admin_request.post( + 'broadcast_message.update_broadcast_message_status', + _data={'areas': ['glasgow'], 'status': BroadcastStatusType.BROADCASTING, 'created_by': str(t.created_by_id)}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=400 + ) + + assert response['errors'] == [{ + 'error': 'ValidationError', + 'message': 'Additional properties are not allowed (areas was unexpected)' + }] + + +def test_update_broadcast_message_status_stores_cancelled_by_and_cancelled_at(admin_request, sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, status=BroadcastStatusType.BROADCASTING) + canceller = create_user('canceller@gov.uk') + + response = admin_request.post( + 'broadcast_message.update_broadcast_message_status', + _data={'status': BroadcastStatusType.CANCELLED, 'created_by': str(canceller.id)}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['status'] == BroadcastStatusType.CANCELLED + assert response['cancelled_at'] is not None + assert response['cancelled_by_id'] == str(canceller.id) + + +def test_update_broadcast_message_status_stores_approved_by_and_approved_at_and_queues_task( + admin_request, + sample_service, + mocker +): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, status=BroadcastStatusType.PENDING_APPROVAL) + approver = create_user('approver@gov.uk') + mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_message.apply_async') + + response = admin_request.post( + 'broadcast_message.update_broadcast_message_status', + _data={'status': BroadcastStatusType.BROADCASTING, 'created_by': str(approver.id)}, + service_id=t.service_id, + broadcast_message_id=bm.id, + _expected_status=200 + ) + + assert response['status'] == BroadcastStatusType.BROADCASTING + assert response['approved_at'] is not None + assert response['approved_by_id'] == str(approver.id) + mock_task.assert_called_once_with(kwargs={'broadcast_message_id': str(bm.id)}, queue='notify-internal-tasks') diff --git a/tests/app/celery/test_broadcast_message_tasks.py b/tests/app/celery/test_broadcast_message_tasks.py new file mode 100644 index 000000000..1622c2a7e --- /dev/null +++ b/tests/app/celery/test_broadcast_message_tasks.py @@ -0,0 +1,61 @@ +import pytest +import requests_mock +from requests import RequestException + +from app.dao.templates_dao import dao_update_template +from app.models import BROADCAST_TYPE, BroadcastStatusType +from app.celery.broadcast_message_tasks import send_broadcast_message +from tests.app.db import create_template, create_broadcast_message + + +def test_send_broadcast_message_sends_data_correctly(sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, areas=['london'], status=BroadcastStatusType.BROADCASTING) + + with requests_mock.Mocker() as request_mock: + request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", json={'valid': 'true'}, status_code=200) + send_broadcast_message(broadcast_message_id=str(bm.id)) + + assert request_mock.call_count == 1 + assert request_mock.request_history[0].method == 'POST' + assert request_mock.request_history[0].headers["Content-type"] == "application/json" + + cbc_json = request_mock.request_history[0].json() + assert cbc_json['template']['id'] == str(t.id) + assert cbc_json['template']['template_type'] == BROADCAST_TYPE + assert cbc_json['broadcast_message']['areas'] == ['london'] + + +def test_send_broadcast_message_sends_old_version_of_template(sample_service): + t = create_template(sample_service, BROADCAST_TYPE, content='first content') + bm = create_broadcast_message(t, areas=['london'], status=BroadcastStatusType.BROADCASTING) + + t.content = 'second content' + dao_update_template(t) + assert t.version == 2 + + with requests_mock.Mocker() as request_mock: + request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", json={'valid': 'true'}, status_code=200) + send_broadcast_message(broadcast_message_id=str(bm.id)) + + assert request_mock.call_count == 1 + assert request_mock.request_history[0].method == 'POST' + assert request_mock.request_history[0].headers["Content-type"] == "application/json" + + cbc_json = request_mock.request_history[0].json() + assert cbc_json['template']['id'] == str(t.id) + assert cbc_json['template']['version'] == 1 + assert cbc_json['template']['content'] == 'first content' + + +def test_send_broadcast_message_errors(sample_service): + t = create_template(sample_service, BROADCAST_TYPE) + bm = create_broadcast_message(t, status=BroadcastStatusType.BROADCASTING) + + with requests_mock.Mocker() as request_mock: + request_mock.post("http://test-cbc-proxy/broadcasts/stub-1", text='503 bad gateway', status_code=503) + # we're not retrying or anything for the moment - but this'll ensure any exception gets logged + with pytest.raises(RequestException) as ex: + send_broadcast_message(broadcast_message_id=str(bm.id)) + + assert ex.value.response.status_code == 503 diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index c587b646e..4b331f279 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -155,20 +155,19 @@ def test_update_billable_units_for_letter_doesnt_update_if_sent_with_test_key(mo @freeze_time('2020-02-17 18:00:00') def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sample_letter_template): + # second class create_notification( template=sample_letter_template, status='created', reference='ref0', created_at=(datetime.now() - timedelta(hours=2)) ) - create_notification( template=sample_letter_template, status='created', reference='ref1', created_at=(datetime.now() - timedelta(hours=3)) ) - create_notification( template=sample_letter_template, status='created', @@ -208,7 +207,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam {'ContentLength': 3}, ]) - results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30)) + results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second') assert mock_s3.call_count == 3 mock_s3.assert_has_calls( @@ -220,6 +219,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam ) assert len(results) == 3 + assert results == [ {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2}, {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1}, @@ -256,7 +256,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ClientError(error_response, "File not found") ]) - results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30)) + results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second') assert mock_head_s3_object.call_count == 2 mock_head_s3_object.assert_has_calls( @@ -275,20 +275,19 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ]) def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mocker, time_to_run_task): with freeze_time("2020-02-17 18:00:00"): + # second class create_notification( template=sample_letter_template, status='created', reference='ref0', created_at=(datetime.now() - timedelta(hours=2)) ) - create_notification( template=sample_letter_template, status='created', reference='ref1', created_at=(datetime.now() - timedelta(hours=3)) ) - create_notification( template=sample_letter_template, status='created', @@ -296,10 +295,38 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock created_at=(datetime.now() - timedelta(days=2)) ) + # first class + create_notification( + template=sample_letter_template, + status='created', + reference='first_class', + created_at=(datetime.now() - timedelta(hours=4)), + postage="first" + ) + + # international + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=3)), + postage="europe" + ) + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=4)), + postage="rest-of-world" + ) + mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[ + {'ContentLength': 1}, + {'ContentLength': 1}, {'ContentLength': 2}, {'ContentLength': 1}, {'ContentLength': 3}, + {'ContentLength': 1}, ]) mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task') @@ -308,26 +335,59 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock with freeze_time(time_to_run_task): collate_letter_pdfs_to_be_sent() - assert len(mock_celery.call_args_list) == 2 + assert len(mock_celery.call_args_list) == 5 assert mock_celery.call_args_list[0] == call( name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ - '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', - '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' + '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.001.k3x_WqC5KhB6e2DWv9Ma.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.1.001.kHh01fdUxT9iEIYUt5Wx.ZIP' }, queue='process-ftp-tasks', compression='zlib' ) assert mock_celery.call_args_list[1] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', + '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' + ], + 'upload_filename': 'NOTIFY.2020-02-17.2.001.k3x_WqC5KhB6e2DWv9Ma.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[2] == call( name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.002.J85cUw-FWlKuAIOcwdLS.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.2.002.J85cUw-FWlKuAIOcwdLS.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[3] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF' + ], + 'upload_filename': 'NOTIFY.2020-02-17.E.001.4YajCZzgzIl7zf8bjWK2.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[4] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', + ], + 'upload_filename': 'NOTIFY.2020-02-17.N.001.eSvP8Ph6EBKhh3k7BSA2.ZIP' }, queue='process-ftp-tasks', compression='zlib' diff --git a/tests/app/db.py b/tests/app/db.py index 08a91e453..e45bff2af 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -59,7 +59,9 @@ from app.models import ( Domain, NotificationHistory, ReturnedLetter, - ServiceContactList + ServiceContactList, + BroadcastMessage, + BroadcastStatusType, ) @@ -984,3 +986,28 @@ def create_service_contact_list( db.session.add(contact_list) db.session.commit() return contact_list + + +def create_broadcast_message( + template, + created_by=None, + personalisation={}, + status=BroadcastStatusType.DRAFT, + starts_at=None, + finishes_at=None, + areas=[], +): + broadcast_message = BroadcastMessage( + service_id=template.service_id, + template_id=template.id, + template_version=template.version, + personalisation=personalisation, + status=BroadcastStatusType.DRAFT, + starts_at=starts_at, + finishes_at=finishes_at, + created_by_id=created_by.id if created_by else template.created_by_id, + areas=areas, + ) + db.session.add(broadcast_message) + db.session.commit() + return broadcast_message