diff --git a/Makefile b/Makefile index 14cae9b0c..4142722ff 100644 --- a/Makefile +++ b/Makefile @@ -306,5 +306,5 @@ cf-push: .PHONY: check-if-migrations-to-run check-if-migrations-to-run: - @echo $(shell python scripts/check_if_new_migration.py) + @echo $(shell python3 scripts/check_if_new_migration.py) diff --git a/app/config.py b/app/config.py index 60d182d91..1cac131e2 100644 --- a/app/config.py +++ b/app/config.py @@ -24,23 +24,6 @@ class QueueNames(object): NOTIFY = 'notify-internal-tasks' PROCESS_FTP = 'process-ftp-tasks' - @staticmethod - def old_queues(): - return [ - 'db-sms', - 'db-email', - 'db-letter', - 'priority', - 'periodic', - 'send-sms', - 'send-email', - 'research-mode', - 'statistics', - 'notify', - 'retry', - 'process-job' - ] - @staticmethod def all_queues(): return [ @@ -124,6 +107,7 @@ class Config(object): SMS_CHAR_COUNT_LIMIT = 495 BRANDING_PATH = '/images/email-template/crests/' TEST_MESSAGE_FILENAME = 'Test message' + ONE_OFF_MESSAGE_FILENAME = 'Report' MAX_VERIFY_CODE_COUNT = 10 NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' @@ -262,8 +246,6 @@ class Development(Config): NOTIFICATION_QUEUE_PREFIX = 'development' DEBUG = True - queues = QueueNames.all_queues() + QueueNames.old_queues() - for queue in QueueNames.all_queues(): Config.CELERY_QUEUES.append( Queue(queue, Exchange('default'), routing_key=queue) @@ -283,9 +265,7 @@ class Test(Config): STATSD_HOST = "localhost" STATSD_PORT = 1000 - queues = QueueNames.all_queues() + QueueNames.old_queues() - - for queue in queues: + for queue in QueueNames.all_queues(): Config.CELERY_QUEUES.append( Queue(queue, Exchange('default'), routing_key=queue) ) diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py new file mode 100644 index 000000000..92f1c79e0 --- /dev/null +++ b/app/dao/inbound_sms_dao.py @@ -0,0 +1,7 @@ +from app import db +from app.dao.dao_utils import transactional + + +@transactional +def dao_create_inbound_sms(inbound_sms): + db.session.add(inbound_sms) diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index b712b0e70..d2ec8c367 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -53,7 +53,8 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id): def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None): query_filter = [ Job.service_id == service_id, - Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'] + Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'], + Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'], ] if limit_days is not None: query_filter.append(cast(Job.created_at, sql_date) >= days_ago(limit_days)) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 4f9e2ee8e..b3b582ed3 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -153,3 +153,70 @@ def rate_multiplier(): (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa ]), Integer()) + + +@statsd(namespace="dao") +def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id): + + billable_units = 0 + total_cost = 0.0 + + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + for rate_boundary in rate_boundaries: + result = db.session.query( + func.sum( + NotificationHistory.billable_units * func.coalesce(NotificationHistory.rate_multiplier, 1) + ).label('billable_units') + ).filter( + NotificationHistory.service_id == service_id, + NotificationHistory.notification_type == 'sms', + NotificationHistory.created_at >= rate_boundary['start_date'], + NotificationHistory.created_at < rate_boundary['end_date'], + NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE) + ) + billable_units_by_rate_boundry = result.scalar() + if billable_units_by_rate_boundry: + billable_units += int(billable_units_by_rate_boundry) + total_cost += int(billable_units_by_rate_boundry) * rate_boundary['rate'] + + return billable_units, total_cost + + +def discover_rate_bounds_for_billing_query(start_date, end_date): + bounds = [] + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + def current_valid_from(index): + return rates[index].valid_from + + def next_valid_from(index): + return rates[index + 1].valid_from + + def current_rate(index): + return rates[index].rate + + def append_rate(rate_start_date, rate_end_date, rate): + bounds.append({ + 'start_date': rate_start_date, + 'end_date': rate_end_date, + 'rate': rate + }) + + if len(rates) == 1: + append_rate(start_date, end_date, current_rate(0)) + return bounds + + for i in range(len(rates)): + # first boundary + if i == 0: + append_rate(start_date, next_valid_from(i), current_rate(i)) + + # last boundary + elif i == (len(rates) - 1): + append_rate(current_valid_from(i), end_date, current_rate(i)) + + # other boundaries + else: + append_rate(current_valid_from(i), next_valid_from(i), current_rate(i)) + + return bounds diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 572e36e63..24c95567a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -3,6 +3,7 @@ from datetime import date, datetime, timedelta from sqlalchemy import asc, func from sqlalchemy.orm import joinedload +from flask import current_app from app import db from app.dao.dao_utils import ( @@ -66,6 +67,12 @@ def dao_fetch_service_by_id(service_id, only_active=False): return query.one() +def dao_fetch_services_by_sms_sender(sms_sender): + return Service.query.filter( + Service.sms_sender == sms_sender + ).all() + + def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False): query = Service.query.filter_by( id=service_id @@ -131,6 +138,12 @@ def dao_fetch_service_by_id_and_user(service_id, user_id): @transactional @version_class(Service) def dao_create_service(service, user, service_id=None, service_permissions=[SMS_TYPE, EMAIL_TYPE]): + # the default property does not appear to work when there is a difference between the sqlalchemy schema and the + # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender + # migration is completed, this code should be able to be removed. + if not service.sms_sender: + service.sms_sender = current_app.config['FROM_NUMBER'] + from app.dao.permissions_dao import permission_dao service.users.append(user) permission_dao.add_default_service_permissions_for_user(user, service) diff --git a/app/models.py b/app/models.py index 153ef57e8..4680b6274 100644 --- a/app/models.py +++ b/app/models.py @@ -146,8 +146,10 @@ class DVLAOrganisation(db.Model): INTERNATIONAL_SMS_TYPE = 'international_sms' INBOUND_SMS_TYPE = 'inbound_sms' +SCHEDULE_NOTIFICATIONS = 'schedule_notifications' -SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE] +SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE, + SCHEDULE_NOTIFICATIONS] class ServicePermissionTypes(db.Model): @@ -188,7 +190,7 @@ class Service(db.Model, Versioned): created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False) reply_to_email_address = db.Column(db.Text, index=False, unique=False, nullable=True) letter_contact_block = db.Column(db.Text, index=False, unique=False, nullable=True) - sms_sender = db.Column(db.String(11), nullable=True, default=lambda: current_app.config['FROM_NUMBER']) + sms_sender = db.Column(db.String(11), nullable=False, default=lambda: current_app.config['FROM_NUMBER']) organisation_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organisation.id'), index=True, nullable=True) organisation = db.relationship('Organisation') dvla_organisation_id = db.Column( @@ -681,6 +683,8 @@ NOTIFICATION_STATUS_TYPES = [ NOTIFICATION_PERMANENT_FAILURE, ] +NOTIFICATION_STATUS_TYPES_NON_BILLABLE = list(set(NOTIFICATION_STATUS_TYPES) - set(NOTIFICATION_STATUS_TYPES_BILLABLE)) + NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type') @@ -977,7 +981,7 @@ INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled'] class ScheduledNotification(db.Model): __tablename__ = 'scheduled_notifications' - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4()) + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notifications.id'), index=True, nullable=False) notification = db.relationship('Notification', uselist=False) scheduled_for = db.Column(db.DateTime, index=False, nullable=False) @@ -1097,6 +1101,12 @@ class Rate(db.Model): rate = db.Column(db.Float(asdecimal=False), nullable=False) notification_type = db.Column(notification_types, index=True, nullable=False) + def __str__(self): + the_string = "{}".format(self.rate) + the_string += " {}".format(self.notification_type) + the_string += " {}".format(self.valid_from) + return the_string + class JobStatistics(db.Model): __tablename__ = 'job_statistics' @@ -1143,6 +1153,29 @@ class JobStatistics(db.Model): return the_string +class InboundSms(db.Model): + __tablename__ = 'inbound_sms' + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) + service = db.relationship('Service', backref='inbound_sms') + + notify_number = db.Column(db.String, nullable=False) # the service's number, that the msg was sent to + user_number = db.Column(db.String, nullable=False) # the end user's number, that the msg was sent from + provider_date = db.Column(db.DateTime) + provider_reference = db.Column(db.String) + _content = db.Column('content', db.String, nullable=False) + + @property + def content(self): + return encryption.decrypt(self._content) + + @content.setter + def content(self, content): + self._content = encryption.encrypt(content) + + class LetterRate(db.Model): __tablename__ = 'letter_rates' diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 08122fb41..cbe17e09e 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -1,8 +1,15 @@ -from flask import Blueprint -from flask import current_app -from flask import request +from urllib.parse import unquote +import iso8601 +from flask import jsonify, Blueprint, current_app, request +from notifications_utils.recipients import normalise_phone_number + +from app import statsd_client +from app.dao.services_dao import dao_fetch_services_by_sms_sender +from app.dao.inbound_sms_dao import dao_create_inbound_sms +from app.models import InboundSms from app.errors import register_errors +from app.utils import convert_bst_to_utc receive_notifications_blueprint = Blueprint('receive_notifications', __name__) register_errors(receive_notifications_blueprint) @@ -10,8 +17,77 @@ register_errors(receive_notifications_blueprint) @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) def receive_mmg_sms(): + """ + { + 'MSISDN': '447123456789' + 'Number': '40604', + 'Message': 'some+uri+encoded+message%3A', + 'ID': 'SOME-MMG-SPECIFIC-ID', + 'DateRecieved': '2017-05-21+11%3A56%3A11' + } + """ post_data = request.get_json() - post_data.pop('MSISDN', None) - current_app.logger.info("Recieve notification form data: {}".format(post_data)) + potential_services = dao_fetch_services_by_sms_sender(post_data['Number']) - return "RECEIVED" + if len(potential_services) != 1: + current_app.logger.error('Inbound number "{}" not associated with exactly one service'.format( + post_data['Number'] + )) + statsd_client.incr('inbound.mmg.failed') + # since this is an issue with our service <-> number mapping, we should still tell MMG that we received + # succesfully + return 'RECEIVED', 200 + + statsd_client.incr('inbound.mmg.succesful') + + service = potential_services[0] + + inbound = create_inbound_mmg_sms_object(service, post_data) + + current_app.logger.info('{} received inbound SMS with reference {}'.format(service.id, inbound.provider_reference)) + + return 'RECEIVED', 200 + + +def format_mmg_message(message): + return unquote(message.replace('+', ' ')) + + +def format_mmg_datetime(date): + """ + We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded + (the same as UTC) + """ + orig_date = format_mmg_message(date) + parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None) + return convert_bst_to_utc(parsed_datetime) + + +def create_inbound_mmg_sms_object(service, json): + message = format_mmg_message(json['Message']) + user_number = normalise_phone_number(json['MSISDN']) + + provider_date = json.get('DateRecieved') + if provider_date: + provider_date = format_mmg_datetime(provider_date) + + inbound = InboundSms( + service=service, + notify_number=service.sms_sender, + user_number=user_number, + provider_date=provider_date, + provider_reference=json.get('ID'), + content=message, + ) + dao_create_inbound_sms(inbound) + return inbound + + +@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) +def receive_firetext_sms(): + post_data = request.form + current_app.logger.info("Received Firetext notification form data: {}".format(post_data)) + + return jsonify({ + "status": "ok" + }), 200 diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 1193d4015..5663a6046 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -6,7 +6,7 @@ from notifications_utils.recipients import ( ) from app.dao import services_dao -from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE +from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE, SCHEDULE_NOTIFICATIONS from app.service.utils import service_allowed_to_send_to from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError from app import redis_store @@ -92,6 +92,7 @@ def check_sms_content_char_count(content_count): raise BadRequestError(message=message) -def service_can_schedule_notification(service): - # TODO: implement once the service permission works. - raise BadRequestError(message="Your service must be invited to schedule notifications via the API.") +def service_can_schedule_notification(service, scheduled_for): + if scheduled_for: + if SCHEDULE_NOTIFICATIONS not in [p.permission for p in service.permissions]: + raise BadRequestError(message="Cannot schedule notifications (this feature is invite-only)") diff --git a/app/service/rest.py b/app/service/rest.py index dfca05c89..24fdcd513 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -10,6 +10,7 @@ from flask import ( ) from sqlalchemy.orm.exc import NoResultFound +from app import redis_store from app.dao import notification_usage_dao, notifications_dao from app.dao.dao_utils import dao_rollback from app.dao.api_key_dao import ( @@ -17,6 +18,8 @@ from app.dao.api_key_dao import ( get_model_api_keys, get_unsigned_secret, expire_api_key) +from app.dao.date_util import get_financial_year +from app.dao.notification_usage_dao import get_total_billable_units_for_sent_sms_notifications_in_date_range from app.dao.services_dao import ( dao_fetch_service_by_id, dao_fetch_all_services, @@ -60,6 +63,7 @@ from app.schemas import ( detailed_service_schema ) from app.utils import pagination_links +from notifications_utils.clients.redis import sms_billable_units_cache_key service_blueprint = Blueprint('service', __name__) @@ -447,6 +451,39 @@ def get_monthly_template_stats(service_id): raise InvalidRequest('Year must be a number', status_code=400) +@service_blueprint.route('//yearly-sms-billable-units') +def get_yearly_sms_billable_units(service_id): + cache_key = sms_billable_units_cache_key(service_id) + cached_billable_sms_units = redis_store.get_all_from_hash(cache_key) + if cached_billable_sms_units: + return jsonify({ + 'billable_sms_units': int(cached_billable_sms_units[b'billable_units']), + 'total_cost': float(cached_billable_sms_units[b'total_cost']) + }) + else: + try: + start_date, end_date = get_financial_year(int(request.args.get('year'))) + except (ValueError, TypeError) as e: + current_app.logger.exception(e) + return jsonify(result='error', message='No valid year provided'), 400 + + billable_units, total_cost = get_total_billable_units_for_sent_sms_notifications_in_date_range( + start_date, + end_date, + service_id) + + cached_values = { + 'billable_units': billable_units, + 'total_cost': total_cost + } + + redis_store.set_hash_and_expire(cache_key, cached_values, expire_in_seconds=60) + return jsonify({ + 'billable_sms_units': billable_units, + 'total_cost': total_cost + }) + + @service_blueprint.route('//yearly-usage') def get_yearly_billing_usage(service_id): try: diff --git a/app/v2/notifications/notification_schemas.py b/app/v2/notifications/notification_schemas.py index 6aa4dcb77..69372c4e1 100644 --- a/app/v2/notifications/notification_schemas.py +++ b/app/v2/notifications/notification_schemas.py @@ -113,7 +113,7 @@ post_sms_request = { "phone_number": {"type": "string", "format": "phone_number"}, "template_id": uuid, "personalisation": personalisation, - "scheduled_for": {"type": "string", "format": "datetime"} + "scheduled_for": {"type": ["string", "null"], "format": "datetime"} }, "required": ["phone_number", "template_id"] } @@ -141,7 +141,7 @@ post_sms_response = { "content": sms_content, "uri": {"type": "string", "format": "uri"}, "template": template, - "scheduled_for": {"type": "string"} + "scheduled_for": {"type": ["string", "null"]} }, "required": ["id", "content", "uri", "template"] } @@ -157,7 +157,7 @@ post_email_request = { "email_address": {"type": "string", "format": "email_address"}, "template_id": uuid, "personalisation": personalisation, - "scheduled_for": {"type": "string", "format": "datetime"} + "scheduled_for": {"type": ["string", "null"], "format": "datetime"} }, "required": ["email_address", "template_id"] } @@ -186,7 +186,7 @@ post_email_response = { "content": email_content, "uri": {"type": "string", "format": "uri"}, "template": template, - "scheduled_for": {"type": "string"} + "scheduled_for": {"type": ["string", "null"]} }, "required": ["id", "content", "uri", "template"] } diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 0b54d98af..47a40a72e 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -35,9 +35,8 @@ def post_notification(notification_type): form = validate(request.get_json(), post_sms_request) scheduled_for = form.get("scheduled_for", None) - if scheduled_for: - if not service_can_schedule_notification(authenticated_service): - return + service_can_schedule_notification(authenticated_service, scheduled_for) + check_rate_limiting(authenticated_service, api_user) form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address'] diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml index a40990194..eccae21b1 100644 --- a/manifest-api-preview.yml +++ b/manifest-api-preview.yml @@ -8,4 +8,4 @@ routes: - route: api.notify.works instances: 1 -memory: 256M +memory: 1G diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 2eaf380a6..7f73a4b54 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -23,34 +23,32 @@ applications: NOTIFY_APP_NAME: delivery-celery-beat - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter,database-tasks - memory: 1G + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks env: NOTIFY_APP_NAME: delivery-worker-database - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode,research-mode-tasks + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks env: NOTIFY_APP_NAME: delivery-worker-research - name: notify-delivery-worker-sender - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email,send-tasks - memory: 1G + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks env: NOTIFY_APP_NAME: delivery-worker-sender - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic,statistics,periodic-tasks,statistics-tasks + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks instances: 1 env: NOTIFY_APP_NAME: delivery-worker-periodic - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority,priority-tasks + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks env: NOTIFY_APP_NAME: delivery-worker-priority - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q process-job,notify,retry,job-tasks,retry-tasks,notify-internal-tasks + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks env: NOTIFY_APP_NAME: delivery-worker diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml index 2bbb3c0dc..492bc6c55 100644 --- a/manifest-delivery-preview.yml +++ b/manifest-delivery-preview.yml @@ -1,4 +1,4 @@ --- inherit: manifest-delivery-base.yml -memory: 256M +memory: 1G diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml index d2c2ba647..53c8d2f12 100644 --- a/manifest-delivery-production.yml +++ b/manifest-delivery-production.yml @@ -3,4 +3,4 @@ inherit: manifest-delivery-base.yml instances: 2 -memory: 768M +memory: 1G diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml index d2c2ba647..53c8d2f12 100644 --- a/manifest-delivery-staging.yml +++ b/manifest-delivery-staging.yml @@ -3,4 +3,4 @@ inherit: manifest-delivery-base.yml instances: 2 -memory: 768M +memory: 1G diff --git a/migrations/versions/0088_add_schedule_serv_perm.py b/migrations/versions/0088_add_schedule_serv_perm.py new file mode 100644 index 000000000..0882c7c94 --- /dev/null +++ b/migrations/versions/0088_add_schedule_serv_perm.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 0088_add_schedule_serv_perm +Revises: 0087_scheduled_notifications +Create Date: 2017-05-26 14:53:18.581320 + +""" + +# revision identifiers, used by Alembic. +revision = '0088_add_schedule_serv_perm' +down_revision = '0087_scheduled_notifications' + +from alembic import op + + +def upgrade(): + op.get_bind() + op.execute("insert into service_permission_types values('schedule_notifications')") + + +def downgrade(): + op.get_bind() + op.execute("delete from service_permissions where permission = 'schedule_notifications'") + op.execute("delete from service_permission_types where name = 'schedule_notifications'") diff --git a/migrations/versions/0089_govuk_sms_sender.py b/migrations/versions/0089_govuk_sms_sender.py new file mode 100644 index 000000000..b69701abd --- /dev/null +++ b/migrations/versions/0089_govuk_sms_sender.py @@ -0,0 +1,25 @@ +"""empty message + +Revision ID: 0089_govuk_sms_sender +Revises: 0088_add_schedule_serv_perm +Create Date: 2017-05-22 13:46:09.584801 + +""" + +# revision identifiers, used by Alembic. +revision = '0089_govuk_sms_sender' +down_revision = '0088_add_schedule_serv_perm' + +from alembic import op + + +def upgrade(): + op.execute("UPDATE services SET sms_sender = 'GOVUK' where sms_sender is null") + op.execute("UPDATE services_history SET sms_sender = 'GOVUK' where sms_sender is null") + op.alter_column('services', 'sms_sender', nullable=False) + op.alter_column('services_history', 'sms_sender', nullable=False) + + +def downgrade(): + op.alter_column('services_history', 'sms_sender', nullable=True) + op.alter_column('services', 'sms_sender', nullable=True) diff --git a/migrations/versions/0090_inbound_sms.py b/migrations/versions/0090_inbound_sms.py new file mode 100644 index 000000000..d02690379 --- /dev/null +++ b/migrations/versions/0090_inbound_sms.py @@ -0,0 +1,37 @@ +"""empty message + +Revision ID: 0090_inbound_sms +Revises: 0089_govuk_sms_sender +Create Date: 2017-05-22 11:28:53.471004 + +""" + +# revision identifiers, used by Alembic. +revision = '0090_inbound_sms' +down_revision = '0089_govuk_sms_sender' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + op.create_table( + 'inbound_sms', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('content', sa.String, nullable=False), + sa.Column('notify_number', sa.String, nullable=False), + sa.Column('user_number', sa.String, nullable=False), + sa.Column('created_at', sa.DateTime, nullable=False), + sa.Column('provider_date', sa.DateTime, nullable=True), + sa.Column('provider_reference', sa.String, nullable=True), + + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_inbound_sms_service_id'), 'inbound_sms', ['service_id'], unique=False) + op.create_index(op.f('ix_inbound_sms_user_number'), 'inbound_sms', ['user_number'], unique=False) + + +def downgrade(): + op.drop_table('inbound_sms') diff --git a/migrations/versions/0088_letter_billing.py b/migrations/versions/0091_letter_billing.py similarity index 92% rename from migrations/versions/0088_letter_billing.py rename to migrations/versions/0091_letter_billing.py index 9c184d10a..7dda9bb70 100644 --- a/migrations/versions/0088_letter_billing.py +++ b/migrations/versions/0091_letter_billing.py @@ -1,7 +1,7 @@ """empty message -Revision ID: 0088_letter_billing -Revises: 0087_scheduled_notifications +Revision ID: 0091_letter_billing +Revises: 0090_inbound_sms Create Date: 2017-05-31 11:43:55.744631 """ @@ -10,8 +10,8 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import postgresql -revision = '0088_letter_billing' -down_revision = '0087_scheduled_notifications' +revision = '0091_letter_billing' +down_revision = '0090_inbound_sms' def upgrade(): diff --git a/requirements.txt b/requirements.txt index 3551937fa..46b02daa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,6 @@ notifications-python-client>=3.1,<3.2 awscli>=1.11,<1.12 awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@17.1.0#egg=notifications-utils==17.1.0 +git+https://github.com/alphagov/notifications-utils.git@17.1.3#egg=notifications-utils==17.1.3 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 034048a32..a9af1afda 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta from functools import partial +import pytest import uuid from freezegun import freeze_time @@ -334,13 +335,24 @@ def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_ assert res.items[1].created_at == datetime(2015, 1, 1, 7) -def test_get_jobs_for_service_doesnt_return_test_messages(notify_db, notify_db_session, sample_template, sample_job): +@pytest.mark.parametrize('file_name', [ + 'Test message', + 'Report', +]) +def test_get_jobs_for_service_doesnt_return_test_messages( + notify_db, + notify_db_session, + sample_template, + sample_job, + file_name, +): test_job = create_job( notify_db, notify_db_session, sample_template.service, sample_template, - original_file_name='Test message') + original_file_name=file_name, + ) jobs = dao_get_jobs_by_service_id(sample_job.service_id).items diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 5252b0f90..d4aec8531 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -1,11 +1,25 @@ import uuid -from datetime import datetime +from datetime import datetime, timedelta + +import pytest from app.dao.date_util import get_financial_year -from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data, - get_monthly_billing_data) -from app.models import Rate +from app.dao.notification_usage_dao import ( + get_rates_for_year, + get_yearly_billing_data, + get_monthly_billing_data, + get_total_billable_units_for_sent_sms_notifications_in_date_range, + discover_rate_bounds_for_billing_query +) +from app.models import ( + Rate, + NOTIFICATION_DELIVERED, + NOTIFICATION_STATUS_TYPES_BILLABLE, + NOTIFICATION_STATUS_TYPES_NON_BILLABLE, + Notification) +from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service from tests.app.db import create_notification +from freezegun import freeze_time def test_get_rates_for_year(notify_db, notify_db_session): @@ -248,3 +262,345 @@ def test_get_monthly_billing_data_with_no_notifications_for_year(notify_db, noti def set_up_rate(notify_db, start_date, value): rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') notify_db.session.add(rate) + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications(notify_db, notify_db_session, sample_service): + set_up_rate(notify_db, datetime(2016, 1, 1), 0.016) + + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=1, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=2, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=3, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, billable_units=4, status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 10 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 0.16 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=1.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=2.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=5.0, status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, notify_db_session, service=sample_service, rate_multiplier=10.0, status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 18 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 45 + + +def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications_for_several_rates( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2) + set_up_rate(notify_db, datetime(2016, 10, 1), 4) + set_up_rate(notify_db, datetime(2017, 1, 1), 6) + + eligble_rate_1 = datetime(2016, 2, 1) + eligble_rate_2 = datetime(2016, 11, 1) + eligble_rate_3 = datetime(2017, 2, 1) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=1.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_1) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=2.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_2) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=5.0, + status=NOTIFICATION_DELIVERED, + created_at=eligble_rate_3) + + start = datetime(2016, 1, 1) + end = datetime(2018, 1, 1) + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 8 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 40 + + +def test_returns_total_billable_units_for_sms_notifications_for_several_rates_where_dates_match_rate_boundary( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2) + set_up_rate(notify_db, datetime(2016, 10, 1), 4) + set_up_rate(notify_db, datetime(2017, 1, 1), 6) + + eligble_rate_1_start = datetime(2016, 1, 1, 0, 0, 0, 0) + eligble_rate_1_end = datetime(2016, 9, 30, 23, 59, 59, 999) + eligble_rate_2_start = datetime(2016, 10, 1, 0, 0, 0, 0) + eligble_rate_2_end = datetime(2016, 12, 31, 23, 59, 59, 999) + eligble_rate_3_start = datetime(2017, 1, 1, 0, 0, 0, 0) + eligble_rate_3_whenever = datetime(2017, 12, 12, 0, 0, 0, 0) + + def make_notification(created_at): + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + rate_multiplier=1.0, + status=NOTIFICATION_DELIVERED, + created_at=created_at) + + make_notification(eligble_rate_1_start) + make_notification(eligble_rate_1_end) + make_notification(eligble_rate_2_start) + make_notification(eligble_rate_2_end) + make_notification(eligble_rate_3_start) + make_notification(eligble_rate_3_whenever) + + start = datetime(2016, 1, 1) + end = datetime(2018, 1, 1) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 6 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 24.0 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_ignoring_letters_and_emails( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + email_template = sample_email_template(notify_db, notify_db_session, service=sample_service) + letter_template = sample_letter_template(sample_service) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + template=email_template, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + template=letter_template, + service=sample_service, + billable_units=2, + status=NOTIFICATION_DELIVERED + ) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_for_only_requested_service( + notify_db, notify_db_session +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + service_1 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + service_2 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + service_3 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4())) + + sample_notification( + notify_db, + notify_db_session, + service=service_1, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + service=service_2, + billable_units=2, + status=NOTIFICATION_DELIVERED) + sample_notification( + notify_db, + notify_db_session, + service=service_3, + billable_units=2, + status=NOTIFICATION_DELIVERED + ) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[1] == 5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_returns_total_billable_units_for_sms_notifications_handling_null_values( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=2, + rate_multiplier=None, + status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5 + + +@pytest.mark.parametrize('billable_units, states', ([ + (len(NOTIFICATION_STATUS_TYPES_BILLABLE), NOTIFICATION_STATUS_TYPES_BILLABLE), + (0, NOTIFICATION_STATUS_TYPES_NON_BILLABLE) +])) +@freeze_time("2016-01-10 12:00:00.000000") +def test_ignores_non_billable_states_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service, billable_units, states +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + for state in states: + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=None, + status=state) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id + )[0] == billable_units + assert get_total_billable_units_for_sent_sms_notifications_in_date_range( + start, end, sample_service.id + )[1] == billable_units * 2.5 + + +@freeze_time("2016-01-10 12:00:00.000000") +def test_restricts_to_time_period_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=1.0, + created_at=datetime.utcnow() - timedelta(minutes=100), + status=NOTIFICATION_DELIVERED) + + sample_notification( + notify_db, + notify_db_session, + service=sample_service, + billable_units=1, + rate_multiplier=1.0, + created_at=datetime.utcnow() - timedelta(minutes=5), + status=NOTIFICATION_DELIVERED) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 1 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 2.5 + + +def test_returns_zero_if_no_matching_rows_when_returning_billable_units_for_sms_notifications( + notify_db, notify_db_session, sample_service +): + set_up_rate(notify_db, datetime(2016, 1, 1), 2.5) + + start = datetime.utcnow() - timedelta(minutes=10) + end = datetime.utcnow() + timedelta(minutes=10) + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 0 + assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 0.0 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_single_relevant_rate(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + set_up_rate(notify_db, datetime(2016, 1, 1), 0.016) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 1 + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == end_date + assert rate_boundaries[0]['rate'] == 0.016 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_two_relevant_rates(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + + rate_1_valid_from = datetime(2016, 1, 1) + rate_2_valid_from = datetime(2017, 1, 1) + + set_up_rate(notify_db, rate_1_valid_from, 0.02) + set_up_rate(notify_db, rate_2_valid_from, 0.04) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 2 + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == rate_2_valid_from + assert rate_boundaries[0]['rate'] == 0.02 + + assert rate_boundaries[1]['start_date'] == rate_2_valid_from + assert rate_boundaries[1]['end_date'] == end_date + assert rate_boundaries[1]['rate'] == 0.04 + + +def test_should_calculate_rate_boundaries_for_billing_query_for_three_relevant_rates(notify_db, notify_db_session): + start_date, end_date = get_financial_year(2016) + rate_1_valid_from = datetime(2016, 1, 1) + rate_2_valid_from = datetime(2017, 1, 1) + rate_3_valid_from = datetime(2017, 2, 1) + + set_up_rate(notify_db, rate_1_valid_from, 0.02) + set_up_rate(notify_db, rate_2_valid_from, 0.04) + set_up_rate(notify_db, rate_3_valid_from, 0.06) + rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date) + assert len(rate_boundaries) == 3 + + assert rate_boundaries[0]['start_date'] == start_date + assert rate_boundaries[0]['end_date'] == rate_2_valid_from + assert rate_boundaries[0]['rate'] == 0.02 + + assert rate_boundaries[1]['start_date'] == rate_2_valid_from + assert rate_boundaries[1]['end_date'] == rate_3_valid_from + assert rate_boundaries[1]['rate'] == 0.04 + + assert rate_boundaries[2]['start_date'] == rate_3_valid_from + assert rate_boundaries[2]['end_date'] == end_date + assert rate_boundaries[2]['rate'] == 0.06 diff --git a/tests/app/dao/test_permissionDAO.py b/tests/app/dao/test_permissions_dao.py similarity index 100% rename from tests/app/dao/test_permissionDAO.py rename to tests/app/dao/test_permissions_dao.py diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index b7b3176dd..800852ca2 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -25,7 +25,8 @@ from app.dao.services_dao import ( fetch_stats_by_date_range_for_all_services, dao_suspend_service, dao_resume_service, - dao_fetch_active_users_for_service + dao_fetch_active_users_for_service, + dao_fetch_services_by_sms_sender ) from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.users_dao import save_model_user @@ -948,3 +949,13 @@ def test_dao_fetch_active_users_for_service_returns_active_only(notify_db, notif users = dao_fetch_active_users_for_service(service.id) assert len(users) == 1 + + +def test_dao_fetch_services_by_sms_sender(notify_db_session): + foo1 = create_service(service_name='a', sms_sender='foo') + foo2 = create_service(service_name='b', sms_sender='foo') + bar = create_service(service_name='c', sms_sender='bar') + + services = dao_fetch_services_by_sms_sender('foo') + + assert {foo1.id, foo2.id} == {x.id for x in services} diff --git a/tests/app/db.py b/tests/app/db.py index 2f637cf1b..76b458dd9 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -39,14 +39,20 @@ def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-off def create_service( - user=None, service_name="Sample service", service_id=None, restricted=False, - service_permissions=[EMAIL_TYPE, SMS_TYPE]): + user=None, + service_name="Sample service", + service_id=None, + restricted=False, + service_permissions=[EMAIL_TYPE, SMS_TYPE], + sms_sender='testing' +): service = Service( name=service_name, message_limit=1000, restricted=restricted, email_from=service_name.lower().replace(' ', '.'), - created_by=user or create_user() + created_by=user or create_user(), + sms_sender=sms_sender ) dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions) return service diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 5cb467ac3..e1a4ec4a2 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -627,7 +627,6 @@ def test_should_set_international_phone_number_to_sent_status( # if 40604 is actually in DB then treat that as if entered manually ('40604', '40604', 'bar'), # 'testing' is the FROM_NUMBER during unit tests - (None, 'testing', 'Sample service: bar'), ('testing', 'testing', 'Sample service: bar'), ]) def test_should_handle_sms_sender_and_prefix_message( diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py index f325fe6f9..5d93e5fd3 100644 --- a/tests/app/notifications/test_receive_notification.py +++ b/tests/app/notifications/test_receive_notification.py @@ -1,14 +1,26 @@ +from datetime import datetime + +import pytest from flask import json +from app.notifications.receive_notifications import ( + format_mmg_message, + format_mmg_datetime, + create_inbound_mmg_sms_object +) -def test_receive_notification_returns_received_to_mmg(client): +from app.models import InboundSms +from tests.app.db import create_service + + +def test_receive_notification_returns_received_to_mmg(client, sample_service): data = {"ID": "1234", "MSISDN": "447700900855", "Message": "Some message to notify", "Trigger": "Trigger?", - "Number": "40604", + "Number": "testing", "Channel": "SMS", - "DateReceived": "2012-06-27-12:33:00" + "DateRecieved": "2012-06-27 12:33:00" } response = client.post(path='/notifications/sms/receive/mmg', data=json.dumps(data), @@ -16,3 +28,79 @@ def test_receive_notification_returns_received_to_mmg(client): assert response.status_code == 200 assert response.get_data(as_text=True) == 'RECEIVED' + + +@pytest.mark.parametrize('message, expected_output', [ + ('abc', 'abc'), + ('', ''), + ('lots+of+words', 'lots of words'), + ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'), + ('x+%2B+y', 'x + y') +]) +def test_format_mmg_message(message, expected_output): + assert format_mmg_message(message) == expected_output + + +@pytest.mark.parametrize('provider_date, expected_output', [ + ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)), + ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 10, 56, 11)) +]) +def test_format_mmg_datetime(provider_date, expected_output): + assert format_mmg_datetime(provider_date) == expected_output + + +def test_create_inbound_mmg_sms_object(sample_service): + sample_service.sms_sender = 'foo' + data = { + 'Message': 'hello+there+%F0%9F%93%A9', + 'Number': 'foo', + 'MSISDN': '07700 900 001', + 'DateRecieved': '2017-01-02+03%3A04%3A05', + 'ID': 'bar', + } + + inbound_sms = create_inbound_mmg_sms_object(sample_service, data) + + assert inbound_sms.service_id == sample_service.id + assert inbound_sms.notify_number == 'foo' + assert inbound_sms.user_number == '7700900001' + assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5) + assert inbound_sms.provider_reference == 'bar' + assert inbound_sms._content != 'hello there 📩' + assert inbound_sms.content == 'hello there 📩' + + +@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services']) +def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number): + create_service(service_name='a', sms_sender='foo') + create_service(service_name='b', sms_sender='foo') + + data = { + 'Message': 'hello', + 'Number': notify_number, + 'MSISDN': '7700900001', + 'DateRecieved': '2017-01-02 03:04:05', + 'ID': 'bar', + } + response = client.post(path='/notifications/sms/receive/mmg', + data=json.dumps(data), + headers=[('Content-Type', 'application/json')]) + + # we still return 'RECEIVED' to MMG + assert response.status_code == 200 + assert response.get_data(as_text=True) == 'RECEIVED' + assert InboundSms.query.count() == 0 + + +def test_receive_notification_returns_received_to_firetext(client): + data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00" + + response = client.post( + path='/notifications/sms/receive/firetext', + data=data, + headers=[('Content-Type', 'application/x-www-form-urlencoded')]) + + assert response.status_code == 200 + result = json.loads(response.get_data(as_text=True)) + + assert result['status'] == 'ok' diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 46adbba1b..f2b0a8247 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -64,9 +64,9 @@ def test_get_service_list_with_only_active_flag(client, service_factory): def test_get_service_list_with_user_id_and_only_active_flag( - client, - sample_user, - service_factory + client, + sample_user, + service_factory ): other_user = create_user(email='foo@bar.gov.uk') @@ -850,7 +850,6 @@ def test_add_existing_user_to_another_service_with_all_permissions(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # check which users part of service user_already_in_service = sample_service.users[0] auth_header = create_authorization_header() @@ -925,7 +924,6 @@ def test_add_existing_user_to_another_service_with_send_permissions(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -969,7 +967,6 @@ def test_add_existing_user_to_another_service_with_manage_permissions(notify_api sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -1013,7 +1010,6 @@ def test_add_existing_user_to_another_service_with_manage_api_keys(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - # they must exist in db first user_to_add = User( name='Invited User', @@ -1054,7 +1050,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api, sample_user): with notify_api.test_request_context(): with notify_api.test_client() as client: - user_to_add = User( name='Invited User', email_address='invited@digital.cabinet-office.gov.uk', @@ -1085,7 +1080,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api, def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_db, notify_db_session, sample_service): with notify_api.test_request_context(): with notify_api.test_client() as client: - existing_user_id = sample_service.users[0].id data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']} @@ -1108,7 +1102,6 @@ def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_d def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db_session, sample_service): with notify_api.test_request_context(): with notify_api.test_client() as client: - incorrect_id = 9876 data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']} @@ -1129,7 +1122,7 @@ def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db def test_remove_user_from_service( - notify_db, notify_db_session, client, sample_user_service_permission + notify_db, notify_db_session, client, sample_user_service_permission ): second_user = create_user(email="new@digital.cabinet-office.gov.uk") # Simulates successfully adding a user to the service @@ -1149,7 +1142,7 @@ def test_remove_user_from_service( def test_remove_non_existant_user_from_service( - client, sample_user_service_permission + client, sample_user_service_permission ): second_user = create_user(email="new@digital.cabinet-office.gov.uk") endpoint = url_for( @@ -1185,13 +1178,11 @@ def test_cannot_remove_only_user_from_service(notify_api, # This test is just here verify get_service_and_api_key_history that is a temp solution # until proper ui is sorted out on admin app def test_get_service_and_api_key_history(notify_api, notify_db, notify_db_session, sample_service): - from tests.app.conftest import sample_api_key as create_sample_api_key api_key = create_sample_api_key(notify_db, notify_db_session, service=sample_service) with notify_api.test_request_context(): with notify_api.test_client() as client: - auth_header = create_authorization_header() response = client.get( path='/service/{}/history'.format(sample_service.id), @@ -1265,12 +1256,12 @@ def test_get_all_notifications_for_service_in_order(notify_api, notify_db, notif ] ) def test_get_all_notifications_for_service_including_ones_made_by_jobs( - client, - notify_db, - notify_db_session, - sample_service, - include_from_test_key, - expected_count_of_notifications + client, + notify_db, + notify_db_session, + sample_service, + include_from_test_key, + expected_count_of_notifications ): with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service) without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service) @@ -1295,10 +1286,10 @@ def test_get_all_notifications_for_service_including_ones_made_by_jobs( def test_get_only_api_created_notifications_for_service( - client, - notify_db, - notify_db_session, - sample_service + client, + notify_db, + notify_db_session, + sample_service ): with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service) without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service) @@ -1315,61 +1306,57 @@ def test_get_only_api_created_notifications_for_service( assert response.status_code == 200 -def test_set_sms_sender_for_service(notify_api, sample_service): - with notify_api.test_request_context(): - with notify_api.test_client() as client: - auth_header = create_authorization_header() - resp = client.get( - '/service/{}'.format(sample_service.id), - headers=[auth_header] - ) - json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert json_resp['data']['name'] == sample_service.name +def test_set_sms_sender_for_service(client, sample_service): + data = { + 'sms_sender': 'elevenchars', + } - data = { - 'sms_sender': 'elevenchars', - } + auth_header = create_authorization_header() - auth_header = create_authorization_header() - - resp = client.post( - '/service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), auth_header] - ) - result = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert result['data']['sms_sender'] == 'elevenchars' + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 200 + assert result['data']['sms_sender'] == 'elevenchars' -def test_set_sms_sender_for_service_rejects_invalid_characters(notify_api, sample_service): - with notify_api.test_request_context(): - with notify_api.test_client() as client: - auth_header = create_authorization_header() - resp = client.get( - '/service/{}'.format(sample_service.id), - headers=[auth_header] - ) - json_resp = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 200 - assert json_resp['data']['name'] == sample_service.name +def test_set_sms_sender_for_service_rejects_invalid_characters(client, sample_service): + data = { + 'sms_sender': 'invalid####', + } - data = { - 'sms_sender': 'invalid####', - } + auth_header = create_authorization_header() - auth_header = create_authorization_header() + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 400 + assert result['result'] == 'error' + assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']} - resp = client.post( - '/service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[('Content-Type', 'application/json'), auth_header] - ) - result = json.loads(resp.get_data(as_text=True)) - assert resp.status_code == 400 - assert result['result'] == 'error' - assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']} + +def test_set_sms_sender_for_service_rejects_null(client, sample_service): + data = { + 'sms_sender': None, + } + + auth_header = create_authorization_header() + + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 400 + assert result['result'] == 'error' + assert result['message'] == {'sms_sender': ['Field may not be null.']} @pytest.mark.parametrize('today_only,stats', [ @@ -1639,11 +1626,11 @@ def test_get_notification_billable_unit_count_missing_year(client, sample_servic ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}), ]) def test_get_service_provider_aggregate_statistics( - client, - sample_service, - query_string, - expected_status, - expected_json, + client, + sample_service, + query_string, + expected_status, + expected_json, ): response = client.get( '/service/{}/fragment/aggregate_statistics{}'.format(sample_service.id, query_string), @@ -1684,11 +1671,11 @@ def test_get_template_stats_by_month_returns_correct_data(notify_db, notify_db_s ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}), ]) def test_get_template_stats_by_month_returns_error_for_incorrect_year( - client, - sample_service, - query_string, - expected_status, - expected_json + client, + sample_service, + query_string, + expected_status, + expected_json ): response = client.get( '/service/{}/notifications/templates/monthly{}'.format(sample_service.id, query_string), @@ -1929,20 +1916,90 @@ def test_update_service_does_not_call_send_notification_when_restricted_not_chan assert not send_notification_mock.called -def test_update_service_works_when_sms_sender_is_null(sample_service, client, mocker): - sample_service.sms_sender = None - data = {'name': 'new name'} +def test_get_yearly_billing_usage_count_returns_400_if_missing_year(client, sample_service): + response = client.get( + '/service/{}/yearly-sms-billable-units'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } - resp = client.post( - 'service/{}'.format(sample_service.id), - data=json.dumps(data), - headers=[create_authorization_header()], - content_type='application/json' + +def test_get_yearly_billing_usage_count_returns_400_if_invalid_year(client, sample_service, mocker): + redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + + response = client.get( + '/service/{}/yearly-sms-billable-units?year=HAHAHAHAH'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + redis_set_mock.assert_not_called() + + +def test_get_yearly_billing_usage_count_returns_200_if_year_provided(client, sample_service, mocker): + redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + + start = datetime.utcnow() + end = datetime.utcnow() + timedelta(minutes=10) + mock_query = mocker.patch( + 'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(100, 200.0) + ) + mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end)) + response = client.get( + '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == { + 'billable_sms_units': 100, + 'total_cost': 200.0 + } + mock_query.assert_called_once_with(start, end, sample_service.id) + mock_year.assert_called_once_with(2016) + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + redis_set_mock.assert_called_once_with( + "{}-sms_billable_units".format(str(sample_service.id)), + {'billable_units': 100, 'total_cost': 200.0}, + expire_in_seconds=60 ) - assert resp.status_code == 200 - # make sure it wasn't changed to not-null under the hood - assert sample_service.sms_sender is None + +def test_get_yearly_billing_usage_count_returns_from_cache_if_present(client, sample_service, mocker): + redis_get_mock = mocker.patch( + 'app.service.rest.redis_store.get_all_from_hash', + return_value={b'total_cost': 100.0, b'billable_units': 50} + ) + redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire') + mock_query = mocker.patch( + 'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(50, 100.0) + ) + + start = datetime.utcnow() + end = datetime.utcnow() + timedelta(minutes=10) + mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end)) + + response = client.get( + '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id), + headers=[create_authorization_header()] + ) + print(response.get_data(as_text=True)) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == { + 'billable_sms_units': 50, + 'total_cost': 100.0 + } + redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id))) + mock_year.assert_not_called() + mock_query.assert_not_called() + redis_set_mock.assert_not_called() def test_search_for_notification_by_to_field_filters_by_status(client, notify_db, notify_db_session): diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index 8d112030a..61f2c4517 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -3,13 +3,14 @@ import uuid import pytest from freezegun import freeze_time -from app.models import Notification, ScheduledNotification +from app.models import Notification, ScheduledNotification, SCHEDULE_NOTIFICATIONS, EMAIL_TYPE, SMS_TYPE from flask import json, current_app from app.models import Notification from app.v2.errors import RateLimitError from tests import create_authorization_header from tests.app.conftest import sample_template as create_sample_template, sample_service +from tests.app.db import create_service, create_template @pytest.mark.parametrize("reference", [None, "reference_from_client"]) @@ -350,26 +351,28 @@ def test_post_sms_should_persist_supplied_sms_number(client, sample_template_wit assert mocked.called -@pytest.mark.skip("Once the service can be invited to schedule notifications we can add this test.") @pytest.mark.parametrize("notification_type, key_send_to, send_to", [("sms", "phone_number", "07700 900 855"), ("email", "email_address", "sample@email.com")]) @freeze_time("2017-05-14 14:00:00") -def test_post_notification_with_scheduled_for(client, sample_template, sample_email_template, +def test_post_notification_with_scheduled_for(client, notify_db, notify_db_session, notification_type, key_send_to, send_to): + service = create_service(service_name=str(uuid.uuid4()), + service_permissions=[EMAIL_TYPE, SMS_TYPE, SCHEDULE_NOTIFICATIONS]) + template = create_template(service=service, template_type=notification_type) data = { key_send_to: send_to, - 'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id), + 'template_id': str(template.id) if notification_type == 'email' else str(template.id), 'scheduled_for': '2017-05-14 14:15' } - auth_header = create_authorization_header(service_id=sample_template.service_id) + auth_header = create_authorization_header(service_id=service.id) response = client.post('/v2/notifications/{}'.format(notification_type), data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) assert response.status_code == 201 resp_json = json.loads(response.get_data(as_text=True)) - scheduled_notification = ScheduledNotification.query.all() + scheduled_notification = ScheduledNotification.query.filter_by(notification_id=resp_json["id"]).all() assert len(scheduled_notification) == 1 assert resp_json["id"] == str(scheduled_notification[0].notification_id) assert resp_json["scheduled_for"] == '2017-05-14 14:15' @@ -379,8 +382,8 @@ def test_post_notification_with_scheduled_for(client, sample_template, sample_em [("sms", "phone_number", "07700 900 855"), ("email", "email_address", "sample@email.com")]) @freeze_time("2017-05-14 14:00:00") -def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_template, sample_email_template, - notification_type, key_send_to, send_to): +def test_post_notification_raises_bad_request_if_service_not_invited_to_schedule( + client, sample_template, sample_email_template, notification_type, key_send_to, send_to): data = { key_send_to: send_to, 'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id), @@ -394,4 +397,4 @@ def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_ assert response.status_code == 400 error_json = json.loads(response.get_data(as_text=True)) assert error_json['errors'] == [ - {"error": "BadRequestError", "message": 'Your service must be invited to schedule notifications via the API.'}] + {"error": "BadRequestError", "message": 'Cannot schedule notifications (this feature is invite-only)'}]