diff --git a/app/__init__.py b/app/__init__.py index 0c2734f77..8d8e18270 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -95,6 +95,7 @@ def register_blueprint(application): from app.notifications.receive_notifications import receive_notifications_blueprint from app.notifications.notifications_ses_callback import ses_callback_blueprint from app.notifications.notifications_sms_callback import sms_callback_blueprint + from app.notifications.notifications_letter_callback import letter_callback_blueprint from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth from app.letters.send_letter_jobs import letter_job @@ -155,6 +156,9 @@ def register_blueprint(application): letter_job.before_request(requires_admin_auth) application.register_blueprint(letter_job) + letter_callback_blueprint.before_request(requires_no_auth) + application.register_blueprint(letter_callback_blueprint) + def register_v2_blueprints(application): from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index b0df268b3..e00591f09 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -44,7 +44,7 @@ def deliver_sms(self, notification_id): except Exception as e: try: current_app.logger.exception( - "RETRY: SMS notification {} failed".format(notification_id) + "SMS notification delivery for id: {} failed".format(notification_id) ) self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) except self.MaxRetriesExceededError: diff --git a/app/config.py b/app/config.py index d2737cc9c..e34dd17d4 100644 --- a/app/config.py +++ b/app/config.py @@ -203,7 +203,6 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' NOTIFICATION_QUEUE_PREFIX = 'development' DEBUG = True - SQLALCHEMY_ECHO = False CELERY_QUEUES = Config.CELERY_QUEUES + [ Queue('db-sms', Exchange('default'), routing_key='db-sms'), Queue('priority', Exchange('default'), routing_key='priority'), diff --git a/app/dao/date_util.py b/app/dao/date_util.py new file mode 100644 index 000000000..92932bb4c --- /dev/null +++ b/app/dao/date_util.py @@ -0,0 +1,18 @@ +from datetime import datetime + +import pytz + + +def get_financial_year(year): + return get_april_fools(year), get_april_fools(year + 1) + + +def get_april_fools(year): + """ + This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, + the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. + :param year: the year to calculate the April 1, 00:00 BST for + :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 + """ + return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( + tzinfo=None) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py new file mode 100644 index 000000000..fb412d849 --- /dev/null +++ b/app/dao/notification_usage_dao.py @@ -0,0 +1,138 @@ +from datetime import datetime + +from sqlalchemy import Float, Integer +from sqlalchemy import func, case, cast +from sqlalchemy import literal_column + +from app import db +from app.dao.date_util import get_financial_year +from app.models import (NotificationHistory, + Rate, + NOTIFICATION_STATUS_TYPES_BILLABLE, + KEY_TYPE_TEST, + SMS_TYPE, + EMAIL_TYPE) +from app.statsd_decorators import statsd +from app.utils import get_london_month_from_utc_column + + +@statsd(namespace="dao") +def get_yearly_billing_data(service_id, year): + start_date, end_date = get_financial_year(year) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + result = [] + for r, n in zip(rates, rates[1:]): + result.append( + sms_yearly_billing_data_query(r.rate, service_id, r.valid_from, n.valid_from)) + + result.append(sms_yearly_billing_data_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) + + result.append(email_yearly_billing_data_query(service_id, start_date, end_date)) + + return sum(result, []) + + +@statsd(namespace="dao") +def get_monthly_billing_data(service_id, year): + start_date, end_date = get_financial_year(year) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + result = [] + for r, n in zip(rates, rates[1:]): + result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from)) + result.extend(sms_billing_data_per_month_query(rates[-1].rate, service_id, rates[-1].valid_from, end_date)) + + return [(datetime.strftime(x[0], "%B"), x[1], x[2], x[3], x[4], x[5]) for x in result] + + +def billing_data_filter(notification_type, start_date, end_date, service_id): + return [ + NotificationHistory.notification_type == notification_type, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.service_id == service_id, + NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE), + NotificationHistory.key_type != KEY_TYPE_TEST + ] + + +def email_yearly_billing_data_query(service_id, start_date, end_date, rate=0): + result = db.session.query( + func.count(NotificationHistory.id), + func.count(NotificationHistory.id), + rate_multiplier(), + NotificationHistory.notification_type, + NotificationHistory.international, + cast(rate, Integer()) + ).filter( + *billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) + ).group_by( + NotificationHistory.notification_type, + rate_multiplier(), + NotificationHistory.international + ).first() + if not result: + return [(0, 0, 1, EMAIL_TYPE, False, 0)] + else: + return [result] + + +def sms_yearly_billing_data_query(rate, service_id, start_date, end_date): + result = db.session.query( + cast(func.sum(NotificationHistory.billable_units * rate_multiplier()), Integer()), + func.sum(NotificationHistory.billable_units), + rate_multiplier(), + NotificationHistory.notification_type, + NotificationHistory.international, + cast(rate, Float()) + ).filter( + *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) + ).group_by( + NotificationHistory.notification_type, + NotificationHistory.international, + rate_multiplier() + ).order_by( + rate_multiplier() + ).all() + + if not result: + return [(0, 0, 1, SMS_TYPE, False, rate)] + else: + return result + + +def get_rates_for_year(start_date, end_date, notification_type): + return Rate.query.filter(Rate.valid_from >= start_date, Rate.valid_from < end_date, + Rate.notification_type == notification_type).order_by(Rate.valid_from).all() + + +def sms_billing_data_per_month_query(rate, service_id, start_date, end_date): + month = get_london_month_from_utc_column(NotificationHistory.created_at) + result = db.session.query( + month, + func.sum(NotificationHistory.billable_units), + rate_multiplier(), + NotificationHistory.international, + NotificationHistory.notification_type, + cast(rate, Float()) + ).filter( + *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) + ).group_by( + NotificationHistory.notification_type, + month, + NotificationHistory.rate_multiplier, + NotificationHistory.international + ).order_by( + month, + rate_multiplier() + ).all() + + return result + + +def rate_multiplier(): + return cast(case([ + (NotificationHistory.rate_multiplier == None, literal_column("'1'")), # noqa + (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier), # noqa + ]), Integer()) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 29632e69d..6bb5337f8 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,5 +1,4 @@ import functools -import pytz from datetime import ( datetime, timedelta, @@ -12,6 +11,7 @@ from sqlalchemy.orm import joinedload from app import db, create_uuid from app.dao import days_ago +from app.dao.date_util import get_financial_year from app.models import ( Service, Notification, @@ -26,8 +26,8 @@ from app.models import ( NOTIFICATION_TEMPORARY_FAILURE, NOTIFICATION_PERMANENT_FAILURE, KEY_TYPE_NORMAL, KEY_TYPE_TEST, - LETTER_TYPE -) + LETTER_TYPE, + NOTIFICATION_SENT) from app.dao.dao_utils import transactional from app.statsd_decorators import statsd @@ -175,11 +175,14 @@ def _update_notification_status(notification, status): def update_notification_status_by_id(notification_id, status): notification = Notification.query.with_lockmode("update").filter( Notification.id == notification_id, - or_(Notification.status == NOTIFICATION_CREATED, + or_( + Notification.status == NOTIFICATION_CREATED, Notification.status == NOTIFICATION_SENDING, - Notification.status == NOTIFICATION_PENDING)).first() + Notification.status == NOTIFICATION_PENDING, + Notification.status == NOTIFICATION_SENT + )).first() - if not notification: + if not notification or notification.status == NOTIFICATION_SENT: return None return _update_notification_status( @@ -193,10 +196,13 @@ def update_notification_status_by_id(notification_id, status): def update_notification_status_by_reference(reference, status): notification = Notification.query.filter( Notification.reference == reference, - or_(Notification.status == NOTIFICATION_SENDING, - Notification.status == NOTIFICATION_PENDING)).first() + or_( + Notification.status == NOTIFICATION_SENDING, + Notification.status == NOTIFICATION_PENDING, + Notification.status == NOTIFICATION_SENT + )).first() - if not notification: + if not notification or notification.status == NOTIFICATION_SENT: return None return _update_notification_status( @@ -237,13 +243,15 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page def get_notification_billable_unit_count_per_month(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) notifications = db.session.query( month, func.sum(NotificationHistory.billable_units) ).filter( NotificationHistory.billable_units != 0, NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( month ).order_by( @@ -404,21 +412,6 @@ def dao_timeout_notifications(timeout_period_in_seconds): return updated -def get_financial_year(year): - return get_april_fools(year), get_april_fools(year + 1) - - -def get_april_fools(year): - """ - This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. - :param year: the year to calculate the April 1, 00:00 BST for - :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 - """ - return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( - tzinfo=None) - - def get_total_sent_notifications_in_date_range(start_date, end_date, notification_type): result = db.session.query( func.count(NotificationHistory.id).label('count') diff --git a/app/dao/provider_details_dao.py b/app/dao/provider_details_dao.py index ff4db1a93..1e2c11727 100644 --- a/app/dao/provider_details_dao.py +++ b/app/dao/provider_details_dao.py @@ -60,6 +60,7 @@ def dao_toggle_sms_provider(identifier): @transactional def dao_switch_sms_provider_to_provider_with_identifier(identifier): new_provider = get_provider_details_by_identifier(identifier) + if provider_is_inactive(new_provider): return @@ -69,7 +70,7 @@ def dao_switch_sms_provider_to_provider_with_identifier(identifier): providers_to_update = [] if conflicting_provider: - providers_to_update = switch_providers(conflicting_provider, new_provider) + switch_providers(conflicting_provider, new_provider) else: current_provider = get_current_provider('sms') if not provider_is_primary(current_provider, new_provider, identifier): @@ -79,10 +80,14 @@ def dao_switch_sms_provider_to_provider_with_identifier(identifier): dao_update_provider_details(provider) -def get_provider_details_by_notification_type(notification_type): - return ProviderDetails.query.filter_by( - notification_type=notification_type - ).order_by(asc(ProviderDetails.priority)).all() +def get_provider_details_by_notification_type(notification_type, supports_international=False): + + filters = [ProviderDetails.notification_type == notification_type] + + if supports_international: + filters.append(ProviderDetails.supports_international == supports_international) + + return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all() @transactional diff --git a/app/dao/rates_dao.py b/app/dao/rates_dao.py deleted file mode 100644 index 47ed83a4e..000000000 --- a/app/dao/rates_dao.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import desc - -from app import db -from app.models import Rate - - -def get_rate_for_type_and_date(notification_type, date_sent): - return db.session.query(Rate).filter(Rate.notification_type == notification_type, - Rate.valid_from <= date_sent - ).order_by(Rate.valid_from.desc() - ).limit(1).first() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 928b3188b..fe25469da 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -229,6 +229,7 @@ def _stats_for_service_query(service_id): def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) sq = db.session.query( NotificationHistory.template_id, NotificationHistory.status, @@ -236,7 +237,9 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) func.count().label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)) + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date + ).group_by( month, NotificationHistory.template_id, @@ -262,6 +265,7 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) def dao_fetch_monthly_historical_stats_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) rows = db.session.query( NotificationHistory.notification_type, NotificationHistory.status, @@ -269,7 +273,8 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year): func.count(NotificationHistory.id).label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( NotificationHistory.notification_type, NotificationHistory.status, diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index f6991c0b0..f9ca7861f 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -15,7 +15,8 @@ from app.dao.provider_details_dao import ( ) from app.celery.research_mode_tasks import send_sms_response, send_email_response from app.dao.templates_dao import dao_get_template_by_id -from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE +from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \ + NOTIFICATION_SENT, NOTIFICATION_SENDING def send_sms_to_provider(notification): @@ -25,7 +26,7 @@ def send_sms_to_provider(notification): return if notification.status == 'created': - provider = provider_to_use(SMS_TYPE, notification.id) + provider = provider_to_use(SMS_TYPE, notification.id, notification.international) current_app.logger.info( "Starting sending SMS {} to provider at {}".format(notification.id, datetime.utcnow()) ) @@ -44,7 +45,7 @@ def send_sms_to_provider(notification): else: try: provider.send_sms( - to=validate_and_format_phone_number(notification.to), + to=validate_and_format_phone_number(notification.to, international=notification.international), content=str(template), reference=str(notification.id), sender=service.sms_sender @@ -54,7 +55,7 @@ def send_sms_to_provider(notification): raise e else: notification.billable_units = template.fragment_count - update_notification(notification, provider) + update_notification(notification, provider, notification.international) current_app.logger.info( "SMS {} sent to provider {} at {}".format(notification.id, provider.get_name(), notification.sent_at) @@ -113,16 +114,19 @@ def send_email_to_provider(notification): statsd_client.timing("email.total-time", delta_milliseconds) -def update_notification(notification, provider): +def update_notification(notification, provider, international=False): notification.sent_at = datetime.utcnow() notification.sent_by = provider.get_name() - notification.status = 'sending' + if international: + notification.status = NOTIFICATION_SENT + else: + notification.status = NOTIFICATION_SENDING dao_update_notification(notification) -def provider_to_use(notification_type, notification_id): +def provider_to_use(notification_type, notification_id, international=False): active_providers_in_order = [ - provider for provider in get_provider_details_by_notification_type(notification_type) if provider.active + p for p in get_provider_details_by_notification_type(notification_type, international) if p.active ] if not active_providers_in_order: @@ -147,11 +151,8 @@ def get_logo_url(base_url, branding_path, logo_file): base_url = parse.urlparse(base_url) netloc = base_url.netloc - if ( - base_url.netloc.startswith('localhost') or - # covers both preview and staging - 'notify.works' in base_url.netloc - ): + # covers both preview and staging + if base_url.netloc.startswith('localhost') or 'notify.works' in base_url.netloc: path = '/static' + branding_path + logo_file else: if base_url.netloc.startswith('www'): diff --git a/app/models.py b/app/models.py index d54f413e4..0ef2de359 100644 --- a/app/models.py +++ b/app/models.py @@ -443,6 +443,7 @@ class ProviderDetails(db.Model): updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True) created_by = db.relationship('User') + supports_international = db.Column(db.Boolean, nullable=False, default=False) class ProviderDetailsHistory(db.Model, HistoryModel): @@ -458,6 +459,7 @@ class ProviderDetailsHistory(db.Model, HistoryModel): updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=True) created_by = db.relationship('User') + supports_international = db.Column(db.Boolean, nullable=False, default=False) JOB_STATUS_PENDING = 'pending' @@ -572,6 +574,7 @@ class VerifyCode(db.Model): NOTIFICATION_CREATED = 'created' NOTIFICATION_SENDING = 'sending' +NOTIFICATION_SENT = 'sent' NOTIFICATION_DELIVERED = 'delivered' NOTIFICATION_PENDING = 'pending' NOTIFICATION_FAILED = 'failed' @@ -586,6 +589,7 @@ NOTIFICATION_STATUS_TYPES_FAILED = [ ] NOTIFICATION_STATUS_TYPES_COMPLETED = [ + NOTIFICATION_SENT, NOTIFICATION_DELIVERED, NOTIFICATION_FAILED, NOTIFICATION_TECHNICAL_FAILURE, @@ -595,6 +599,7 @@ NOTIFICATION_STATUS_TYPES_COMPLETED = [ NOTIFICATION_STATUS_TYPES_BILLABLE = [ NOTIFICATION_SENDING, + NOTIFICATION_SENT, NOTIFICATION_DELIVERED, NOTIFICATION_FAILED, NOTIFICATION_TECHNICAL_FAILURE, @@ -605,6 +610,7 @@ NOTIFICATION_STATUS_TYPES_BILLABLE = [ NOTIFICATION_STATUS_TYPES = [ NOTIFICATION_CREATED, NOTIFICATION_SENDING, + NOTIFICATION_SENT, NOTIFICATION_DELIVERED, NOTIFICATION_PENDING, NOTIFICATION_FAILED, @@ -660,6 +666,12 @@ class Notification(db.Model): foreign(template_version) == remote(TemplateHistory.version) )) + client_reference = db.Column(db.String, index=True, nullable=True) + + international = db.Column(db.Boolean, nullable=False, default=False) + phone_prefix = db.Column(db.String, nullable=True) + rate_multiplier = db.Column(db.Float(asdecimal=False), nullable=True) + @property def personalisation(self): if self._personalisation: @@ -740,7 +752,8 @@ class Notification(db.Model): 'permanent-failure': 'Email address doesn’t exist', 'delivered': 'Delivered', 'sending': 'Sending', - 'created': 'Sending' + 'created': 'Sending', + 'sent': 'Delivered' }, 'sms': { 'failed': 'Failed', @@ -749,7 +762,8 @@ class Notification(db.Model): 'permanent-failure': 'Phone number doesn’t exist', 'delivered': 'Delivered', 'sending': 'Sending', - 'created': 'Sending' + 'created': 'Sending', + 'sent': 'Sent internationally' }, 'letter': { 'failed': 'Failed', @@ -758,7 +772,8 @@ class Notification(db.Model): 'permanent-failure': 'Permanent failure', 'delivered': 'Delivered', 'sending': 'Sending', - 'created': 'Sending' + 'created': 'Sending', + 'sent': 'Delivered' } }[self.template.template_type].get(self.status, self.status) @@ -833,6 +848,10 @@ class NotificationHistory(db.Model, HistoryModel): reference = db.Column(db.String, nullable=True, index=True) client_reference = db.Column(db.String, nullable=True) + international = db.Column(db.Boolean, nullable=False, default=False) + phone_prefix = db.Column(db.String, nullable=True) + rate_multiplier = db.Column(db.Float(asdecimal=False), nullable=True) + @classmethod def from_original(cls, notification): history = super().from_original(notification) @@ -952,5 +971,5 @@ class Rate(db.Model): id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) valid_from = db.Column(db.DateTime, nullable=False) - rate = db.Column(db.Numeric(), nullable=False) + rate = db.Column(db.Float(asdecimal=False), nullable=False) notification_type = db.Column(notification_types, index=True, nullable=False) diff --git a/app/notifications/notifications_letter_callback.py b/app/notifications/notifications_letter_callback.py new file mode 100644 index 000000000..f7c357055 --- /dev/null +++ b/app/notifications/notifications_letter_callback.py @@ -0,0 +1,39 @@ +from datetime import datetime + +from flask import ( + Blueprint, + jsonify, + request, + current_app, + json +) + +from app import statsd_client +from app.clients.email.aws_ses import get_aws_responses +from app.dao import ( + notifications_dao +) + +from app.notifications.process_client_response import validate_callback_data + +letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__) + +from app.errors import ( + register_errors, + InvalidRequest +) + +register_errors(letter_callback_blueprint) + + +@letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST']) +def process_letter_response(): + try: + dvla_request = json.loads(request.data) + current_app.logger.info(dvla_request) + return jsonify( + result="success", message="DVLA callback succeeded" + ), 200 + except ValueError: + error = "DVLA callback failed: invalid json" + raise InvalidRequest(error, status_code=400) diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 31d8b8c1b..e8510ba63 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -2,6 +2,11 @@ from datetime import datetime from flask import current_app +from notifications_utils.recipients import ( + get_international_phone_info, + validate_and_format_phone_number +) + from app import redis_store from app.celery import provider_tasks from notifications_utils.clients import redis @@ -24,22 +29,23 @@ def check_placeholders(template_object): raise BadRequestError(fields=[{'template': message}], message=message) -def persist_notification(template_id, - template_version, - recipient, - service, - personalisation, - notification_type, - api_key_id, - key_type, - created_at=None, - job_id=None, - job_row_number=None, - reference=None, - client_reference=None, - notification_id=None, - simulated=False): - # if simulated create a Notification model to return but do not persist the Notification to the dB +def persist_notification( + template_id, + template_version, + recipient, + service, + personalisation, + notification_type, + api_key_id, + key_type, + created_at=None, + job_id=None, + job_row_number=None, + reference=None, + client_reference=None, + notification_id=None, + simulated=False +): notification = Notification( id=notification_id, template_id=template_id, @@ -57,6 +63,15 @@ def persist_notification(template_id, client_reference=client_reference, reference=reference ) + + if notification_type == SMS_TYPE: + formatted_recipient = validate_and_format_phone_number(recipient, international=True) + recipient_info = get_international_phone_info(formatted_recipient) + notification.international = recipient_info.international + notification.phone_prefix = recipient_info.country_prefix + notification.rate_multiplier = recipient_info.billable_units + + # if simulated create a Notification model to return but do not persist the Notification to the dB if not simulated: dao_create_notification(notification) if key_type != KEY_TYPE_TEST: @@ -98,6 +113,10 @@ def send_notification_to_queue(notification, research_mode, queue=None): def simulated_recipient(to_address, notification_type): - return (to_address in current_app.config['SIMULATED_SMS_NUMBERS'] - if notification_type == SMS_TYPE - else to_address in current_app.config['SIMULATED_EMAIL_ADDRESSES']) + if notification_type == SMS_TYPE: + formatted_simulated_numbers = [ + validate_and_format_phone_number(number) for number in current_app.config['SIMULATED_SMS_NUMBERS'] + ] + return to_address in formatted_simulated_numbers + else: + return to_address in current_app.config['SIMULATED_EMAIL_ADDRESSES'] diff --git a/app/notifications/rest.py b/app/notifications/rest.py index e325e3ae0..eb395736b 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -2,8 +2,7 @@ from flask import ( Blueprint, jsonify, request, - current_app, - json + current_app ) from app import api_user @@ -14,10 +13,6 @@ from app.dao import ( ) from app.models import KEY_TYPE_TEAM, PRIORITY from app.models import SMS_TYPE -from app.notifications.process_client_response import ( - validate_callback_data, - process_sms_client_response -) from app.notifications.process_notifications import (persist_notification, send_notification_to_queue, simulated_recipient) @@ -35,6 +30,8 @@ from app.schemas import ( from app.service.utils import service_allowed_to_send_to from app.utils import pagination_links, get_template_instance +from notifications_utils.recipients import get_international_phone_info + notifications = Blueprint('notifications', __name__) from app.errors import ( @@ -104,13 +101,15 @@ def send_notification(notification_type): notification_form, errors = ( sms_template_notification_schema if notification_type == SMS_TYPE else email_notification_schema ).load(request.get_json()) + if errors: raise InvalidRequest(errors, status_code=400) check_rate_limiting(service, api_user) - template = templates_dao.dao_get_template_by_id_and_service_id(template_id=notification_form['template'], - service_id=service.id) + template = templates_dao.dao_get_template_by_id_and_service_id( + template_id=notification_form['template'], + service_id=service.id) check_template_is_for_notification_type(notification_type, template.template_type) check_template_is_active(template) @@ -118,12 +117,14 @@ def send_notification(notification_type): template_object = create_template_object_for_notification(template, notification_form.get('personalisation', {})) _service_allowed_to_send_to(notification_form, service) + if notification_type == SMS_TYPE: + _service_can_send_internationally(service, notification_form['to']) # Do not persist or send notification to the queue if it is a simulated recipient simulated = simulated_recipient(notification_form['to'], notification_type) notification_model = persist_notification(template_id=template.id, template_version=template.version, - recipient=notification_form['to'], + recipient=request.get_json()['to'], service=service, personalisation=notification_form.get('personalisation', None), notification_type=notification_type, @@ -160,6 +161,16 @@ def get_notification_return_data(notification_id, notification, template): return output +def _service_can_send_internationally(service, number): + international_phone_info = get_international_phone_info(number) + + if international_phone_info.international and not service.can_send_international_sms: + raise InvalidRequest( + {'to': ["Cannot send to international mobile numbers"]}, + status_code=400 + ) + + def _service_allowed_to_send_to(notification, service): if not service_allowed_to_send_to(notification['to'], service, api_user.key_type): if api_user.key_type == KEY_TYPE_TEAM: diff --git a/app/notifications/validators.py b/app/notifications/validators.py index bbde4648e..a3cf5872f 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -1,5 +1,9 @@ from flask import current_app -from notifications_utils.recipients import validate_and_format_phone_number, validate_and_format_email_address +from notifications_utils.recipients import ( + validate_and_format_phone_number, + validate_and_format_email_address, + get_international_phone_info +) from app.dao import services_dao from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE @@ -61,8 +65,17 @@ def service_can_send_to_recipient(send_to, key_type, service): def validate_and_format_recipient(send_to, key_type, service, notification_type): service_can_send_to_recipient(send_to, key_type, service) + if notification_type == SMS_TYPE: - return validate_and_format_phone_number(number=send_to) + international_phone_info = get_international_phone_info(send_to) + + if international_phone_info.international and not service.can_send_international_sms: + raise BadRequestError(message="Cannot send to international mobile numbers") + + return validate_and_format_phone_number( + number=send_to, + international=international_phone_info.international + ) else: return validate_and_format_email_address(email_address=send_to) diff --git a/app/schema_validation/__init__.py b/app/schema_validation/__init__.py index 674d83ff8..72767c881 100644 --- a/app/schema_validation/__init__.py +++ b/app/schema_validation/__init__.py @@ -11,7 +11,7 @@ def validate(json_to_validate, schema): @format_checker.checks('phone_number', raises=InvalidPhoneError) def validate_schema_phone_number(instance): if instance is not None: - validate_phone_number(instance) + validate_phone_number(instance, international=True) return True @format_checker.checks('email_address', raises=InvalidEmailError) diff --git a/app/schemas.py b/app/schemas.py index 510d7e59e..2a42c7d15 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -324,13 +324,13 @@ class SmsNotificationSchema(NotificationSchema): @validates('to') def validate_to(self, value): try: - validate_phone_number(value) + validate_phone_number(value, international=True) except InvalidPhoneError as error: raise ValidationError('Invalid phone number: {}'.format(error)) @post_load def format_phone_number(self, item): - item['to'] = validate_and_format_phone_number(item['to']) + item['to'] = validate_and_format_phone_number(item['to'], international=True) return item diff --git a/app/service/rest.py b/app/service/rest.py index 26722373c..25991f30d 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,4 +1,5 @@ import itertools +import json from datetime import datetime from flask import ( @@ -8,6 +9,7 @@ from flask import ( ) from sqlalchemy.orm.exc import NoResultFound +from app.dao import notification_usage_dao from app.dao.dao_utils import dao_rollback from app.dao.api_key_dao import ( save_model_api_key, @@ -411,3 +413,38 @@ def get_monthly_template_stats(service_id): )) except ValueError: raise InvalidRequest('Year must be a number', status_code=400) + + +@service_blueprint.route('//yearly-usage') +def get_yearly_billing_usage(service_id): + try: + year = int(request.args.get('year')) + results = notification_usage_dao.get_yearly_billing_data(service_id, year) + json_result = [{"credits": x[0], + "billing_units": x[1], + "rate_multiplier": x[2], + "notification_type": x[3], + "international": x[4], + "rate": x[5] + } for x in results] + return json.dumps(json_result) + + except TypeError: + return jsonify(result='error', message='No valid year provided'), 400 + + +@service_blueprint.route('//monthly-usage') +def get_yearly_monthly_usage(service_id): + try: + year = int(request.args.get('year')) + results = notification_usage_dao.get_monthly_billing_data(service_id, year) + json_results = [{"month": x[0], + "billing_units": x[1], + "rate_multiplier": x[2], + "international": x[3], + "notification_type": x[4], + "rate": x[5] + } for x in results] + return json.dumps(json_results) + except TypeError: + return jsonify(result='error', message='No valid year provided'), 400 diff --git a/app/service/statistics.py b/app/service/statistics.py index b9cd985f2..8ebf8434b 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -48,7 +48,7 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row): update_dict['requested'] += row.count - if row.status == 'delivered': + if row.status in ('delivered', 'sent'): update_dict['delivered'] += row.count elif row.status in ('failed', 'technical-failure', 'temporary-failure', 'permanent-failure'): update_dict['failed'] += row.count diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 78aa4fa91..347884ed8 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -31,6 +31,7 @@ def post_notification(notification_type): form = validate(request.get_json(), post_email_request) else: form = validate(request.get_json(), post_sms_request) + service = services_dao.dao_fetch_service_by_id(api_user.service_id) check_rate_limiting(service, api_user) @@ -48,7 +49,7 @@ def post_notification(notification_type): notification = persist_notification(template_id=template.id, template_version=template.version, - recipient=send_to, + recipient=form_send_to, service=service, personalisation=form.get('personalisation', None), notification_type=notification_type, diff --git a/migrations/versions/0076_add_intl_flag_to_provider.py b/migrations/versions/0076_add_intl_flag_to_provider.py new file mode 100644 index 000000000..921bb85dd --- /dev/null +++ b/migrations/versions/0076_add_intl_flag_to_provider.py @@ -0,0 +1,27 @@ +"""empty message + +Revision ID: 0076_add_intl_flag_to_provider +Revises: 0075_create_rates_table +Create Date: 2017-04-25 09:44:13.194164 + +""" + +# revision identifiers, used by Alembic. +revision = '0076_add_intl_flag_to_provider' +down_revision = '0075_create_rates_table' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('provider_details', sa.Column('supports_international', sa.Boolean(), nullable=False, server_default=sa.false())) + op.add_column('provider_details_history', sa.Column('supports_international', sa.Boolean(), nullable=False, server_default=sa.false())) + + op.execute("UPDATE provider_details SET supports_international=True WHERE identifier='mmg'") + op.execute("UPDATE provider_details_history SET supports_international=True WHERE identifier='mmg'") + + +def downgrade(): + op.drop_column('provider_details_history', 'supports_international') + op.drop_column('provider_details', 'supports_international') diff --git a/migrations/versions/0077_add_intl_notification.py b/migrations/versions/0077_add_intl_notification.py new file mode 100644 index 000000000..0e1d513c5 --- /dev/null +++ b/migrations/versions/0077_add_intl_notification.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: 0077_add_intl_notification +Revises: 0076_add_intl_flag_to_provider +Create Date: 2017-04-25 11:34:43.229494 + +""" + +# revision identifiers, used by Alembic. +revision = '0077_add_intl_notification' +down_revision = '0076_add_intl_flag_to_provider' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('notification_history', sa.Column('international', sa.Boolean(), nullable=True)) + op.add_column('notification_history', sa.Column('phone_prefix', sa.String(), nullable=True)) + op.add_column('notification_history', sa.Column('rate_multiplier', sa.Numeric(), nullable=True)) + op.add_column('notifications', sa.Column('international', sa.Boolean(), nullable=True)) + op.add_column('notifications', sa.Column('phone_prefix', sa.String(), nullable=True)) + op.add_column('notifications', sa.Column('rate_multiplier', sa.Numeric(), nullable=True)) + + +def downgrade(): + op.drop_column('notifications', 'rate_multiplier') + op.drop_column('notifications', 'phone_prefix') + op.drop_column('notifications', 'international') + op.drop_column('notification_history', 'rate_multiplier') + op.drop_column('notification_history', 'phone_prefix') + op.drop_column('notification_history', 'international') diff --git a/migrations/versions/0078_add_sent_notification_status.py b/migrations/versions/0078_add_sent_notification_status.py new file mode 100644 index 000000000..a52526c60 --- /dev/null +++ b/migrations/versions/0078_add_sent_notification_status.py @@ -0,0 +1,60 @@ +"""empty message + +Revision ID: 0078_sent_notification_status +Revises: 0077_add_intl_notification +Create Date: 2017-04-24 16:55:20.731069 + +""" + +# revision identifiers, used by Alembic. +revision = '0078_sent_notification_status' +down_revision = '0077_add_intl_notification' + +from alembic import op +import sqlalchemy as sa + +enum_name = 'notify_status_type' +tmp_name = 'tmp_' + enum_name + +old_options = ( + 'created', + 'sending', + 'delivered', + 'pending', + 'failed', + 'technical-failure', + 'temporary-failure', + 'permanent-failure' +) +new_options = old_options + ('sent',) + +old_type = sa.Enum(*old_options, name=enum_name) +new_type = sa.Enum(*new_options, name=enum_name) + +alter_str = 'ALTER TABLE {table} ALTER COLUMN status TYPE {enum} USING status::text::notify_status_type ' + +def upgrade(): + op.execute('ALTER TYPE {enum} RENAME TO {tmp_name}'.format(enum=enum_name, tmp_name=tmp_name)) + + new_type.create(op.get_bind()) + op.execute(alter_str.format(table='notifications', enum=enum_name)) + op.execute(alter_str.format(table='notification_history', enum=enum_name)) + + op.execute('DROP TYPE ' + tmp_name) + + +def downgrade(): + op.execute('ALTER TYPE {enum} RENAME TO {tmp_name}'.format(enum=enum_name, tmp_name=tmp_name)) + + # Convert 'sent' template into 'sending' + update_str = "UPDATE {table} SET status='sending' where status='sent'" + + op.execute(update_str.format(table='notifications')) + op.execute(update_str.format(table='notification_history')) + + old_type.create(op.get_bind()) + + op.execute(alter_str.format(table='notifications', enum=enum_name)) + op.execute(alter_str.format(table='notification_history', enum=enum_name)) + + op.execute('DROP TYPE ' + tmp_name) diff --git a/requirements.txt b/requirements.txt index b95191623..3bc64fbc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,6 @@ notifications-python-client>=3.1,<3.2 awscli>=1.11,<1.12 awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@16.0.1#egg=notifications-utils==16.0.1 +git+https://github.com/alphagov/notifications-utils.git@16.1.3#egg=notifications-utils==16.1.3 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/tests/app/conftest.py b/tests/app/conftest.py index fab72ae7e..fc9748870 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -123,7 +123,8 @@ def sample_service( user=None, restricted=False, limit=1000, - email_from=None + email_from=None, + can_send_international_sms=False ): if user is None: user = create_user() @@ -136,6 +137,7 @@ def sample_service( 'email_from': email_from, 'created_by': user, 'letter_contact_block': 'London,\nSW1A 1AA', + 'can_send_international_sms': can_send_international_sms } service = Service.query.filter_by(name=service_name).first() if not service: @@ -442,7 +444,8 @@ def sample_notification( api_key_id=None, key_type=KEY_TYPE_NORMAL, sent_by=None, - client_reference=None + client_reference=None, + rate_multiplier=1.0 ): if created_at is None: created_at = datetime.utcnow() @@ -479,7 +482,8 @@ def sample_notification( 'key_type': key_type, 'sent_by': sent_by, 'updated_at': created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None, - 'client_reference': client_reference + 'client_reference': client_reference, + 'rate_multiplier': rate_multiplier } if job_row_number is not None: data['job_row_number'] = job_row_number diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py new file mode 100644 index 000000000..d6be85da2 --- /dev/null +++ b/tests/app/dao/test_date_utils.py @@ -0,0 +1,13 @@ +from app.dao.date_util import get_financial_year, get_april_fools + + +def test_get_financial_year(): + start, end = get_financial_year(2000) + assert str(start) == '2000-03-31 23:00:00' + assert str(end) == '2001-03-31 23:00:00' + + +def test_get_april_fools(): + april_fools = get_april_fools(2016) + assert str(april_fools) == '2016-03-31 23:00:00' + assert april_fools.tzinfo is None diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index b3598fd7f..f01b631ba 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -43,6 +43,7 @@ def test_should_get_all_statuses_for_notifications_associated_with_job( notification(status='technical-failure') notification(status='temporary-failure') notification(status='permanent-failure') + notification(status='sent') results = dao_get_notification_outcomes_for_job(sample_service.id, sample_job.id) assert [(row.count, row.status) for row in results] == [ @@ -53,7 +54,8 @@ def test_should_get_all_statuses_for_notifications_associated_with_job( (1, 'failed'), (1, 'technical-failure'), (1, 'temporary-failure'), - (1, 'permanent-failure') + (1, 'permanent-failure'), + (1, 'sent') ] diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 99d1200ec..b5461e036 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -3,7 +3,6 @@ import uuid from functools import partial import pytest - from freezegun import freeze_time from sqlalchemy.exc import SQLAlchemyError, IntegrityError @@ -15,10 +14,10 @@ from app.models import ( TemplateStatistics, NOTIFICATION_STATUS_TYPES, NOTIFICATION_STATUS_TYPES_FAILED, + NOTIFICATION_SENT, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, - KEY_TYPE_TEST -) + KEY_TYPE_TEST) from app.dao.notifications_dao import ( dao_create_notification, @@ -39,8 +38,6 @@ from app.dao.notifications_dao import ( update_notification_status_by_reference, dao_delete_notifications_and_history_by_id, dao_timeout_notifications, - get_financial_year, - get_april_fools, is_delivery_slow_for_provider, dao_update_notifications_sent_to_dvla) @@ -352,6 +349,30 @@ def test_should_update_status_by_id_if_created(notify_db, notify_db_session): assert updated.status == 'failed' +def test_should_not_update_status_by_reference_if_in_sent_status(notify_db, notify_db_session): + notification = sample_notification( + notify_db, + notify_db_session, + status=NOTIFICATION_SENT, + reference='foo' + ) + + update_notification_status_by_reference('foo', 'failed') + assert Notification.query.get(notification.id).status == NOTIFICATION_SENT + + +def test_should_not_update_status_by_id_if_in_sent_status(notify_db, notify_db_session): + notification = sample_notification( + notify_db, + notify_db_session, + status=NOTIFICATION_SENT + ) + + update_notification_status_by_id(notification.id, 'failed') + + assert Notification.query.get(notification.id).status == NOTIFICATION_SENT + + def test_should_not_update_status_by_reference_if_not_sending(notify_db, notify_db_session): notification = sample_notification(notify_db, notify_db_session, status='created', reference='reference') assert Notification.query.get(notification.id).status == 'created' @@ -1334,18 +1355,6 @@ def test_should_exclude_test_key_notifications_by_default( assert len(all_notifications) == 1 -def test_get_financial_year(): - start, end = get_financial_year(2000) - assert str(start) == '2000-03-31 23:00:00' - assert str(end) == '2001-03-31 23:00:00' - - -def test_get_april_fools(): - april_fools = get_april_fools(2016) - assert str(april_fools) == '2016-03-31 23:00:00' - assert april_fools.tzinfo is None - - @pytest.mark.parametrize('notification_type', ['sms', 'email']) def test_get_total_sent_notifications_in_date_range_returns_only_in_date_range( notify_db, diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py new file mode 100644 index 000000000..0c3ddc952 --- /dev/null +++ b/tests/app/dao/test_notification_usage_dao.py @@ -0,0 +1,172 @@ +import uuid +from datetime import datetime + +from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data, + get_monthly_billing_data) +from app.models import Rate +from tests.app.db import create_notification + + +def test_get_rates_for_year(notify_db, notify_db_session): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.50) + set_up_rate(notify_db, datetime(2017, 6, 1), 1.75) + rates = get_rates_for_year(datetime(2016, 4, 1), datetime(2017, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" + assert rates[0].rate == 1.50 + rates = get_rates_for_year(datetime(2017, 4, 1), datetime(2018, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" + assert rates[0].rate == 1.75 + + +def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + set_up_rate(notify_db, datetime(2016, 6, 1), 1.58) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3, rate_multiplier=2, international=True, phone_prefix="1") + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=5) + create_notification(template=sample_email_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=0) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=6) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 4 + assert results[0] == (3, 3, 1, 'sms', False, 1.4) + assert results[1] == (9, 9, 1, 'sms', False, 1.58) + assert results[2] == (6, 3, 2, 'sms', True, 1.58) + assert results[3] == (2, 2, 1, 'email', False, 0) + + +def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 22, 59, 59), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=5) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=7) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (15, 15, 1, 'sms', False, 1.4) + assert results[1] == (0, 0, 1, 'email', False, 0) + + +def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + create_notification(template=sample_email_template, created_at=datetime(2016, 7, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2016, 10, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=0) + + results = get_yearly_billing_data(sample_email_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (0, 0, 1, 'sms', False, 1.4) + assert results[1] == (2, 2, 1, 'email', False, 0) + + +def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3, rate_multiplier=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3, rate_multiplier=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 30), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=4) + + create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=0) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) + results = get_monthly_billing_data(sample_template.service_id, 2016) + assert len(results) == 4 + # (billable_units, rate_multiplier, international, type, rate) + assert results[0] == ('April', 1, 1, False, 'sms', 1.4) + assert results[1] == ('May', 2, 1, False, 'sms', 1.4) + assert results[2] == ('July', 7, 1, False, 'sms', 1.4) + assert results[3] == ('July', 6, 2, False, 'sms', 1.4) + + +def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template, + sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + set_up_rate(notify_db, datetime(2016, 6, 5), 1.75) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 6, 1), sent_at=datetime(2016, 6, 1), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 6, 15), sent_at=datetime(2016, 6, 15), + status='sending', billable_units=4) + create_notification(template=sample_email_template, created_at=datetime(2016, 8, 22), + sent_at=datetime(2016, 7, 22), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2016, 8, 30), + sent_at=datetime(2016, 7, 22), + status='sending', billable_units=0) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) + results = get_monthly_billing_data(sample_template.service_id, 2016) + assert len(results) == 4 + assert results[0] == ('April', 1, 1, False, 'sms', 1.4) + assert results[1] == ('May', 2, 1, False, 'sms', 1.4) + assert results[2] == ('June', 3, 1, False, 'sms', 1.4) + assert results[3] == ('June', 4, 1, False, 'sms', 1.75) + + +def test_get_monthly_billing_data_with_no_notifications_for_year(notify_db, notify_db_session, sample_template, + sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + results = get_monthly_billing_data(sample_template.service_id, 2016) + assert len(results) == 0 + + +def set_up_rate(notify_db, start_date, value): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') + notify_db.session.add(rate) diff --git a/tests/app/dao/test_provider_details_dao.py b/tests/app/dao/test_provider_details_dao.py index 5092bd1b9..ed3f0e1f0 100644 --- a/tests/app/dao/test_provider_details_dao.py +++ b/tests/app/dao/test_provider_details_dao.py @@ -35,16 +35,23 @@ def test_can_get_all_providers(restore_provider_details): assert len(get_provider_details()) == 5 -def test_can_get_sms_providers(restore_provider_details): +def test_can_get_sms_non_international_providers(restore_provider_details): sms_providers = get_provider_details_by_notification_type('sms') assert len(sms_providers) == 3 assert all('sms' == prov.notification_type for prov in sms_providers) -def test_can_get_sms_providers_in_order_of_priority(restore_provider_details): - providers = get_provider_details_by_notification_type('sms') +def test_can_get_sms_international_providers(restore_provider_details): + sms_providers = get_provider_details_by_notification_type('sms', True) + assert len(sms_providers) == 1 + assert all('sms' == prov.notification_type for prov in sms_providers) + assert all(prov.supports_international for prov in sms_providers) - assert providers[0].priority < providers[1].priority < providers[2].priority + +def test_can_get_sms_providers_in_order_of_priority(restore_provider_details): + providers = get_provider_details_by_notification_type('sms', False) + + assert providers[0].priority < providers[1].priority def test_can_get_email_providers_in_order_of_priority(restore_provider_details): diff --git a/tests/app/dao/test_provider_statistics_dao.py b/tests/app/dao/test_provider_statistics_dao.py index e3249ec64..54e429b0e 100644 --- a/tests/app/dao/test_provider_statistics_dao.py +++ b/tests/app/dao/test_provider_statistics_dao.py @@ -26,8 +26,8 @@ def test_get_fragment_count_separates_sms_and_email(notify_db, sample_template, def test_get_fragment_count_filters_on_status(notify_db, sample_template): for status in NOTIFICATION_STATUS_TYPES: noti_hist(notify_db, sample_template, status=status) - # sending, delivered, failed, technical-failure, temporary-failure, permanent-failure - assert get_fragment_count(sample_template.service_id)['sms_count'] == 6 + # sending, sent, delivered, failed, technical-failure, temporary-failure, permanent-failure + assert get_fragment_count(sample_template.service_id)['sms_count'] == 7 def test_get_fragment_count_filters_on_service_id(notify_db, sample_template, service_factory): diff --git a/tests/app/dao/test_rates_dao.py b/tests/app/dao/test_rates_dao.py deleted file mode 100644 index 33f68a671..000000000 --- a/tests/app/dao/test_rates_dao.py +++ /dev/null @@ -1,18 +0,0 @@ -from datetime import datetime - -from decimal import Decimal - -from app.dao.rates_dao import get_rate_for_type_and_date - - -def test_get_rate_for_type_and_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime.utcnow()) - assert rate.rate == Decimal("1.58") - - rate = get_rate_for_type_and_date('sms', datetime(2016, 6, 1)) - assert rate.rate == Decimal("1.65") - - -def test_get_rate_for_type_and_date_early_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime(2014, 6, 1)) - assert not rate diff --git a/tests/app/db.py b/tests/app/db.py index 195a77ea7..b5838d693 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -72,7 +72,10 @@ def create_notification( api_key_id=None, key_type=KEY_TYPE_NORMAL, sent_by=None, - client_reference=None + client_reference=None, + rate_multiplier=None, + international=False, + phone_prefix=None ): if created_at is None: created_at = datetime.utcnow() @@ -103,7 +106,10 @@ def create_notification( 'sent_by': sent_by, 'updated_at': updated_at, 'client_reference': client_reference, - 'job_row_number': job_row_number + 'job_row_number': job_row_number, + 'rate_multiplier': rate_multiplier, + 'international': international, + 'phone_prefix': phone_prefix } notification = Notification(**data) dao_create_notification(notification) diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 2894d75e9..0e35ad413 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -4,11 +4,12 @@ from collections import namedtuple from unittest.mock import ANY import pytest -from notifications_utils.recipients import validate_phone_number, format_phone_number +from notifications_utils.recipients import validate_and_format_phone_number import app -from app import mmg_client +from app import mmg_client, firetext_client from app.dao import (provider_details_dao, notifications_dao) +from app.dao.provider_details_dao import dao_switch_sms_provider_to_provider_with_identifier from app.delivery import send_to_providers from app.models import ( Notification, @@ -18,7 +19,7 @@ from app.models import ( KEY_TYPE_TEAM, BRANDING_ORG, BRANDING_BOTH, -) + ProviderDetails) from tests.app.db import create_service, create_template, create_notification @@ -69,7 +70,7 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist( ) mmg_client.send_sms.assert_called_once_with( - to=format_phone_number(validate_phone_number("+447234123123")), + to=validate_and_format_phone_number("+447234123123"), content="Sample service: Hello Jo\nHere is some HTML & entities", reference=str(db_notification.id), sender=None @@ -151,7 +152,7 @@ def test_send_sms_should_use_template_version_from_notification_not_latest( ) mmg_client.send_sms.assert_called_once_with( - to=format_phone_number(validate_phone_number("+447234123123")), + to=validate_and_format_phone_number("+447234123123"), content="Sample service: This is a template:\nwith a newline", reference=str(db_notification.id), sender=None @@ -254,7 +255,7 @@ def test_should_send_sms_sender_from_service_if_present( ) mmg_client.send_sms.assert_called_once_with( - to=format_phone_number(validate_phone_number("+447234123123")), + to=validate_and_format_phone_number("+447234123123"), content="This is a template:\nwith a newline", reference=str(db_notification.id), sender=sample_service.sms_sender @@ -471,3 +472,103 @@ def test_should_update_billable_units_according_to_research_mode_and_key_type(no ) assert sample_notification.billable_units == billable_units + + +def test_should_send_sms_to_international_providers( + restore_provider_details, + sample_sms_template_with_html, + sample_user, + mocker +): + mocker.patch('app.provider_details.switch_providers.get_user_by_id', return_value=sample_user) + + dao_switch_sms_provider_to_provider_with_identifier('firetext') + + db_notification_uk = create_notification( + template=sample_sms_template_with_html, + to_field="+447234123999", + personalisation={"name": "Jo"}, + status='created', + international=False) + + db_notification_international = create_notification( + template=sample_sms_template_with_html, + to_field="+447234123111", + personalisation={"name": "Jo"}, + status='created', + international=True) + + mocker.patch('app.mmg_client.send_sms') + mocker.patch('app.firetext_client.send_sms') + + send_to_providers.send_sms_to_provider( + db_notification_uk + ) + + firetext_client.send_sms.assert_called_once_with( + to="447234123999", + content=ANY, + reference=str(db_notification_uk.id), + sender=None + ) + + send_to_providers.send_sms_to_provider( + db_notification_international + ) + + mmg_client.send_sms.assert_called_once_with( + to="447234123111", + content=ANY, + reference=str(db_notification_international.id), + sender=None + ) + + notification_uk = Notification.query.filter_by(id=db_notification_uk.id).one() + notification_int = Notification.query.filter_by(id=db_notification_international.id).one() + + assert notification_uk.status == 'sending' + assert notification_uk.sent_by == 'firetext' + assert notification_int.status == 'sent' + assert notification_int.sent_by == 'mmg' + + +def test_should_send_international_sms_with_formatted_phone_number( + notify_db, + sample_template, + mocker +): + notification = create_notification( + template=sample_template, + to_field="+6011-17224412", + international=True + ) + + send_notification_mock = mocker.patch('app.mmg_client.send_sms') + mocker.patch('app.delivery.send_to_providers.send_sms_response') + + send_to_providers.send_sms_to_provider( + notification + ) + + assert send_notification_mock.called is True + + +def test_should_set_international_phone_number_to_sent_status( + notify_db, + sample_template, + mocker +): + notification = create_notification( + template=sample_template, + to_field="+6011-17224412", + international=True + ) + + mocker.patch('app.mmg_client.send_sms') + mocker.patch('app.delivery.send_to_providers.send_sms_response') + + send_to_providers.send_sms_to_provider( + notification + ) + + assert notification.status == 'sent' diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index e5a3cc453..e05d52b3a 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -205,7 +205,6 @@ def test_should_not_create_scheduled_job_more_then_24_hours_hence(notify_api, sa auth_header = create_authorization_header() headers = [('Content-Type', 'application/json'), auth_header] - print(json.dumps(data)) response = client.post( path, data=json.dumps(data), @@ -240,7 +239,6 @@ def test_should_not_create_scheduled_job_in_the_past(notify_api, sample_template auth_header = create_authorization_header() headers = [('Content-Type', 'application/json'), auth_header] - print(json.dumps(data)) response = client.post( path, data=json.dumps(data), diff --git a/tests/app/notifications/rest/test_callbacks.py b/tests/app/notifications/rest/test_callbacks.py index c24bb76d3..780076b34 100644 --- a/tests/app/notifications/rest/test_callbacks.py +++ b/tests/app/notifications/rest/test_callbacks.py @@ -13,6 +13,16 @@ from app.models import NotificationStatistics from tests.app.conftest import sample_notification as create_sample_notification +def test_dvla_callback_should_not_need_auth(client): + data = json.dumps({"somekey": "somevalue"}) + response = client.post( + path='/notifications/letter/dvla', + data=data, + headers=[('Content-Type', 'application/json')]) + + assert response.status_code == 200 + + def test_firetext_callback_should_not_need_auth(client, mocker): mocker.patch('app.statsd_client.incr') response = client.post( diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index 9599d2f9a..ab527e06e 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -21,8 +21,8 @@ from tests.app.conftest import ( sample_email_template as create_sample_email_template, sample_template as create_sample_template, sample_service_whitelist as create_sample_service_whitelist, - sample_api_key as create_sample_api_key -) + sample_api_key as create_sample_api_key, + sample_service) from app.models import Template from app.errors import InvalidRequest @@ -1093,3 +1093,75 @@ def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded( assert not persist_mock.called assert not deliver_mock.called + + +def test_should_allow_store_original_number_on_sms_notification(client, sample_template, mocker): + mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch('app.encryption.encrypt', return_value="something_encrypted") + + data = { + 'to': '+(44) 7700-900 855', + 'template': str(sample_template.id) + } + + auth_header = create_authorization_header(service_id=sample_template.service_id) + + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + response_data = json.loads(response.data)['data'] + notification_id = response_data['notification']['id'] + + mocked.assert_called_once_with([notification_id], queue='send-sms') + assert response.status_code == 201 + assert notification_id + notifications = Notification.query.all() + assert len(notifications) == 1 + assert '+(44) 7700-900 855' == notifications[0].to + + +def test_should_not_allow_international_number_on_sms_notification(client, sample_template, mocker): + mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch('app.encryption.encrypt', return_value="something_encrypted") + + data = { + 'to': '20-12-1234-1234', + 'template': str(sample_template.id) + } + + auth_header = create_authorization_header(service_id=sample_template.service_id) + + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + assert not mocked.called + assert response.status_code == 400 + error_json = json.loads(response.get_data(as_text=True)) + assert error_json['result'] == 'error' + assert error_json['message']['to'][0] == 'Cannot send to international mobile numbers' + + +def test_should_allow_international_number_on_sms_notification(client, notify_db, notify_db_session, mocker): + mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch('app.encryption.encrypt', return_value="something_encrypted") + + service = sample_service(notify_db, notify_db_session, can_send_international_sms=True) + template = create_sample_template(notify_db, notify_db_session, service=service) + + data = { + 'to': '20-12-1234-1234', + 'template': str(template.id) + } + + auth_header = create_authorization_header(service_id=service.id) + + response = client.post( + path='/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + assert response.status_code == 201 diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index 9e428a0ab..f697047e7 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -1,7 +1,7 @@ import datetime import uuid - import pytest + from boto3.exceptions import Boto3Error from sqlalchemy.exc import SQLAlchemyError from freezegun import freeze_time @@ -9,10 +9,13 @@ from collections import namedtuple from app.models import Template, Notification, NotificationHistory from app.notifications import SendNotificationToQueueError -from app.notifications.process_notifications import (create_content_for_notification, - persist_notification, - send_notification_to_queue, - simulated_recipient) +from app.notifications.process_notifications import ( + create_content_for_notification, + persist_notification, + send_notification_to_queue, + simulated_recipient +) +from notifications_utils.recipients import validate_and_format_phone_number, validate_and_format_email_address from app.utils import cache_key_for_service_template_counter from app.v2.errors import BadRequestError from tests.app.conftest import sample_api_key as create_api_key @@ -169,18 +172,21 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker) 'app.notifications.process_notifications.redis_store.get_all_from_hash') n_id = uuid.uuid4() created_at = datetime.datetime(2016, 11, 11, 16, 8, 18) - persist_notification(template_id=sample_job.template.id, - template_version=sample_job.template.version, - recipient='+447111111111', - service=sample_job.service, - personalisation=None, notification_type='sms', - api_key_id=sample_api_key.id, - key_type=sample_api_key.key_type, - created_at=created_at, - job_id=sample_job.id, - job_row_number=10, - client_reference="ref from client", - notification_id=n_id) + persist_notification( + template_id=sample_job.template.id, + template_version=sample_job.template.version, + recipient='+447111111111', + service=sample_job.service, + personalisation=None, + notification_type='sms', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + created_at=created_at, + job_id=sample_job.id, + job_row_number=10, + client_reference="ref from client", + notification_id=n_id + ) assert Notification.query.count() == 1 assert NotificationHistory.query.count() == 1 persisted_notification = Notification.query.all()[0] @@ -192,6 +198,9 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker) mock_service_template_cache.assert_called_once_with(cache_key_for_service_template_counter(sample_job.service_id)) assert persisted_notification.client_reference == "ref from client" assert persisted_notification.reference is None + assert persisted_notification.international is False + assert persisted_notification.phone_prefix == '44' + assert persisted_notification.rate_multiplier == 1 @freeze_time("2016-01-01 11:09:00.061258") @@ -255,22 +264,97 @@ def test_send_notification_to_queue_throws_exception_deletes_notification(sample assert NotificationHistory.query.count() == 0 -@pytest.mark.parametrize("to_address, notification_type, expected", - [("+447700900000", "sms", True), - ("+447700900111", "sms", True), - ("+447700900222", "sms", True), - ("simulate-delivered@notifications.service.gov.uk", "email", True), - ("simulate-delivered-2@notifications.service.gov.uk", "email", True), - ("simulate-delivered-3@notifications.service.gov.uk", "email", True), - ("07515896969", "sms", False), - ("valid_email@test.com", "email", False)]) +@pytest.mark.parametrize("to_address, notification_type, expected", [ + ("+447700900000", "sms", True), + ("+447700900111", "sms", True), + ("+447700900222", "sms", True), + ("07700900000", "sms", True), + ("7700900111", "sms", True), + ("simulate-delivered@notifications.service.gov.uk", "email", True), + ("simulate-delivered-2@notifications.service.gov.uk", "email", True), + ("simulate-delivered-3@notifications.service.gov.uk", "email", True), + ("07515896969", "sms", False), + ("valid_email@test.com", "email", False) +]) def test_simulated_recipient(notify_api, to_address, notification_type, expected): - # The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES'] - # and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue. - # SIMULATED_EMAIL_ADDRESSES = ('simulate-delivered@notifications.service.gov.uk', - # 'simulate-delivered-2@notifications.service.gov.uk', - # 'simulate-delivered-2@notifications.service.gov.uk') - # SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') + """ + The values where the expected = 'research-mode' are listed in the config['SIMULATED_EMAIL_ADDRESSES'] + and config['SIMULATED_SMS_NUMBERS']. These values should result in using the research mode queue. + SIMULATED_EMAIL_ADDRESSES = ( + 'simulate-delivered@notifications.service.gov.uk', + 'simulate-delivered-2@notifications.service.gov.uk', + 'simulate-delivered-2@notifications.service.gov.uk' + ) + SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222') + """ + formatted_address = None - actual = simulated_recipient(to_address, notification_type) - assert actual == expected + if notification_type == 'email': + formatted_address = validate_and_format_email_address(to_address) + else: + formatted_address = validate_and_format_phone_number(to_address) + + is_simulated_address = simulated_recipient(formatted_address, notification_type) + + assert is_simulated_address == expected + + +@pytest.mark.parametrize('recipient, expected_international, expected_prefix, expected_units', [ + ('7900900123', False, '44', 1), # UK + ('+447900900123', False, '44', 1), # UK + ('07700900222', False, '44', 1), # UK + ('73122345678', True, '7', 1), # Russia + ('360623400400', True, '36', 3)] # Hungary +) +def test_persist_notification_with_international_info_stores_correct_info( + sample_job, + sample_api_key, + mocker, + recipient, + expected_international, + expected_prefix, + expected_units +): + persist_notification( + template_id=sample_job.template.id, + template_version=sample_job.template.version, + recipient=recipient, + service=sample_job.service, + personalisation=None, + notification_type='sms', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + job_id=sample_job.id, + job_row_number=10, + client_reference="ref from client" + ) + persisted_notification = Notification.query.all()[0] + + assert persisted_notification.international is expected_international + assert persisted_notification.phone_prefix == expected_prefix + assert persisted_notification.rate_multiplier == expected_units + + +def test_persist_notification_with_international_info_does_not_store_for_email( + sample_job, + sample_api_key, + mocker +): + persist_notification( + template_id=sample_job.template.id, + template_version=sample_job.template.version, + recipient='foo@bar.com', + service=sample_job.service, + personalisation=None, + notification_type='email', + api_key_id=sample_api_key.id, + key_type=sample_api_key.key_type, + job_id=sample_job.id, + job_row_number=10, + client_reference="ref from client" + ) + persisted_notification = Notification.query.all()[0] + + assert persisted_notification.international is False + assert persisted_notification.phone_prefix is None + assert persisted_notification.rate_multiplier is None diff --git a/tests/app/notifications/test_validators.py b/tests/app/notifications/test_validators.py index 4ce51c24b..ea964fc43 100644 --- a/tests/app/notifications/test_validators.py +++ b/tests/app/notifications/test_validators.py @@ -8,7 +8,10 @@ from app.notifications.validators import ( check_template_is_active, service_can_send_to_recipient, check_sms_content_char_count, - check_service_over_api_rate_limit) + check_service_over_api_rate_limit, + validate_and_format_recipient +) + from app.v2.errors import ( BadRequestError, TooManyRequestsError, @@ -337,3 +340,19 @@ def test_should_not_rate_limit_if_limiting_is_disabled( check_service_over_api_rate_limit(service, api_key) assert not app.redis_store.exceeded_rate_limit.called + + +@pytest.mark.parametrize('key_type', ['test', 'normal']) +def test_rejects_api_calls_with_international_numbers_if_service_does_not_allow_int_sms(sample_service, key_type): + with pytest.raises(BadRequestError) as e: + validate_and_format_recipient('20-12-1234-1234', key_type, sample_service, 'sms') + assert e.value.status_code == 400 + assert e.value.message == 'Cannot send to international mobile numbers' + assert e.value.fields == [] + + +@pytest.mark.parametrize('key_type', ['test', 'normal']) +def test_allows_api_calls_with_international_numbers_if_service_does_allow_int_sms(sample_service, key_type): + sample_service.can_send_international_sms = True + result = validate_and_format_recipient('20-12-1234-1234', key_type, sample_service, 'sms') + assert result == '201212341234' diff --git a/tests/app/provider_details/test_rest.py b/tests/app/provider_details/test_rest.py index 15e2381be..36a40d2a0 100644 --- a/tests/app/provider_details/test_rest.py +++ b/tests/app/provider_details/test_rest.py @@ -47,7 +47,7 @@ def test_get_provider_details_contains_correct_fields(client, notify_db): allowed_keys = { "id", "created_by", "display_name", "identifier", "priority", 'notification_type', - "active", "version", "updated_at" + "active", "version", "updated_at", "supports_international" } assert allowed_keys == set(json_resp[0].keys()) @@ -116,7 +116,7 @@ def test_get_provider_versions_contains_correct_fields(client, notify_db): allowed_keys = { "id", "created_by", "display_name", "identifier", "priority", 'notification_type', - "active", "version", "updated_at" + "active", "version", "updated_at", "supports_international" } assert allowed_keys == set(json_resp[0].keys()) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 43edbc040..f2150df1a 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -10,8 +10,9 @@ from freezegun import freeze_time from app.dao.users_dao import save_model_user from app.dao.services_dao import dao_remove_user_from_service -from app.models import User, Organisation, DVLA_ORG_LAND_REGISTRY +from app.models import User, Organisation, DVLA_ORG_LAND_REGISTRY, Rate from tests import create_authorization_header +from tests.app.db import create_template from tests.app.conftest import ( sample_service as create_service, sample_service_permission as create_service_permission, @@ -224,6 +225,7 @@ def test_create_service(client, sample_user): assert json_resp['data']['name'] == 'created service' assert not json_resp['data']['research_mode'] assert not json_resp['data']['can_send_letters'] + assert not json_resp['data']['can_send_international_sms'] def test_should_not_create_service_with_missing_user_id_field(notify_api, fake_uuid): @@ -419,10 +421,12 @@ def test_update_service_flags(notify_api, sample_service): assert json_resp['data']['name'] == sample_service.name assert json_resp['data']['research_mode'] is False assert json_resp['data']['can_send_letters'] is False + assert json_resp['data']['can_send_international_sms'] is False data = { 'research_mode': True, - 'can_send_letters': True + 'can_send_letters': True, + 'can_send_international_sms': True, } auth_header = create_authorization_header() @@ -436,6 +440,7 @@ def test_update_service_flags(notify_api, sample_service): assert resp.status_code == 200 assert result['data']['research_mode'] is True assert result['data']['can_send_letters'] is True + assert result['data']['can_send_international_sms'] is True def test_update_service_research_mode_throws_validation_error(notify_api, sample_service): @@ -1505,3 +1510,107 @@ def test_get_template_stats_by_month_returns_error_for_incorrect_year( ) assert response.status_code == expected_status assert json.loads(response.get_data(as_text=True)) == expected_json + + +def test_get_yearly_billing_usage(client, notify_db, notify_db_session): + rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=1.58, notification_type='sms') + notify_db.session.add(rate) + notification = create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), + sent_at=datetime(2016, 6, 5), + status='sending') + response = client.get( + '/service/{}/yearly-usage?year=2016'.format(notification.service_id), + headers=[create_authorization_header()] + ) + assert response.status_code == 200 + + assert json.loads(response.get_data(as_text=True)) == [{'credits': 1, + 'billing_units': 1, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'international': False, + 'rate': 1.58}, + {'credits': 0, + 'billing_units': 0, + 'rate_multiplier': 1, + 'notification_type': 'email', + 'international': False, + 'rate': 0}] + + +def test_get_yearly_billing_usage_returns_400_if_missing_year(client, sample_service): + response = client.get( + '/service/{}/yearly-usage'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } + + +def test_get_monthly_billing_usage(client, notify_db, notify_db_session, sample_service): + rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=1.58, notification_type='sms') + notify_db.session.add(rate) + notification = create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), + sent_at=datetime(2016, 6, 5), + status='sending') + create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), + sent_at=datetime(2016, 6, 5), + status='sending', rate_multiplier=2) + create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 7, 5), + sent_at=datetime(2016, 7, 5), + status='sending') + + template = create_template(sample_service, template_type='email') + create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), + sent_at=datetime(2016, 6, 5), + status='sending', + template=template) + response = client.get( + '/service/{}/monthly-usage?year=2016'.format(notification.service_id), + headers=[create_authorization_header()] + ) + assert response.status_code == 200 + actual = json.loads(response.get_data(as_text=True)) + assert len(actual) == 3 + assert actual == [{'month': 'June', + 'international': False, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'rate': 1.58, + 'billing_units': 1}, + {'month': 'June', + 'international': False, + 'rate_multiplier': 2, + 'notification_type': 'sms', + 'rate': 1.58, + 'billing_units': 1}, + {'month': 'July', + 'international': False, + 'rate_multiplier': 1, + 'notification_type': 'sms', + 'rate': 1.58, + 'billing_units': 1}] + + +def test_get_monthly_billing_usage_returns_400_if_missing_year(client, sample_service): + response = client.get( + '/service/{}/monthly-usage'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 400 + assert json.loads(response.get_data(as_text=True)) == { + 'message': 'No valid year provided', 'result': 'error' + } + + +def test_get_monthly_billing_usage_returns_empty_list_if_no_notifications(client, notify_db, sample_service): + rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=1.58, notification_type='sms') + notify_db.session.add(rate) + response = client.get( + '/service/{}/monthly-usage?year=2016'.format(sample_service.id), + headers=[create_authorization_header()] + ) + assert response.status_code == 200 + assert json.loads(response.get_data(as_text=True)) == [] diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index fd686be18..9c35ad9c7 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -30,6 +30,11 @@ StatsRow = collections.namedtuple('row', ('notification_type', 'status', 'count' StatsRow('email', 'temporary-failure', 1), StatsRow('email', 'permanent-failure', 1), ], [4, 0, 4], [0, 0, 0], [0, 0, 0]), + 'convert_sent_to_delivered': ([ + StatsRow('sms', 'sending', 1), + StatsRow('sms', 'delivered', 1), + StatsRow('sms', 'sent', 1), + ], [0, 0, 0], [3, 2, 0], [0, 0, 0]), }) def test_format_statistics(stats, email_counts, sms_counts, letter_counts): diff --git a/tests/app/test_model.py b/tests/app/test_model.py index ba0920745..14fa5609f 100644 --- a/tests/app/test_model.py +++ b/tests/app/test_model.py @@ -111,6 +111,7 @@ def test_notification_for_csv_returns_correct_job_row_number(notify_db, notify_d ('email', 'permanent-failure', 'Email address doesn’t exist'), ('sms', 'temporary-failure', 'Phone not accepting messages right now'), ('sms', 'permanent-failure', 'Phone number doesn’t exist'), + ('sms', 'sent', 'Sent internationally'), ('letter', 'permanent-failure', 'Permanent failure'), ('letter', 'delivered', 'Delivered') ]) diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 1e08115a9..08f510bbd 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -230,7 +230,7 @@ def test_send_user_code_for_sms_with_optional_to_field(client, """ Tests POST endpoint /user//sms-code with optional to field """ - to_number = '+441119876757' + to_number = '+447119876757' mocked = mocker.patch('app.user.rest.create_secret_code', return_value='11111') mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') auth_header = create_authorization_header() diff --git a/tests/app/v2/notifications/test_get_notifications.py b/tests/app/v2/notifications/test_get_notifications.py index 5c2c31af3..6e3b72ed2 100644 --- a/tests/app/v2/notifications/test_get_notifications.py +++ b/tests/app/v2/notifications/test_get_notifications.py @@ -307,8 +307,8 @@ def test_get_all_notifications_filter_by_status_invalid_status(client, sample_no assert json_response['status_code'] == 400 assert len(json_response['errors']) == 1 - assert json_response['errors'][0]['message'] == "status elephant is not one of [created, sending, delivered, " \ - "pending, failed, technical-failure, temporary-failure, permanent-failure]" + assert json_response['errors'][0]['message'] == "status elephant is not one of [created, sending, sent, " \ + "delivered, pending, failed, technical-failure, temporary-failure, permanent-failure]" def test_get_all_notifications_filter_by_multiple_statuses(client, notify_db, notify_db_session): diff --git a/tests/app/v2/notifications/test_notification_schemas.py b/tests/app/v2/notifications/test_notification_schemas.py index 44a743a1a..04c32e1c7 100644 --- a/tests/app/v2/notifications/test_notification_schemas.py +++ b/tests/app/v2/notifications/test_notification_schemas.py @@ -27,7 +27,7 @@ def test_get_notifications_request_invalid_statuses( invalid_statuses, valid_statuses ): partial_error_status = "is not one of " \ - "[created, sending, delivered, pending, failed, " \ + "[created, sending, sent, delivered, pending, failed, " \ "technical-failure, temporary-failure, permanent-failure]" with pytest.raises(ValidationError) as e: @@ -74,7 +74,7 @@ def test_get_notifications_request_invalid_statuses_and_template_types(): error_messages = [error['message'] for error in errors] for invalid_status in ["elephant", "giraffe"]: - assert "status {} is not one of [created, sending, delivered, " \ + assert "status {} is not one of [created, sending, sent, delivered, " \ "pending, failed, technical-failure, temporary-failure, permanent-failure]".format( invalid_status ) in error_messages diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index fd6b0cc78..14f8d6d2e 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -4,7 +4,7 @@ from flask import json from app.models import Notification from app.v2.errors import RateLimitError from tests import create_authorization_header -from tests.app.conftest import sample_template as create_sample_template +from tests.app.conftest import sample_template as create_sample_template, sample_service @pytest.mark.parametrize("reference", [None, "reference_from_client"]) @@ -201,6 +201,9 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori notification_type, key_send_to, send_to): + + mocker.patch('app.celery.provider_tasks.deliver_{}.apply_async'.format(notification_type)) + sample = create_sample_template( notify_db, notify_db_session, @@ -273,3 +276,74 @@ def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded( assert not persist_mock.called assert not deliver_mock.called + + +def test_post_sms_notification_returns_400_if_not_allowed_to_send_int_sms(client, sample_service, sample_template): + data = { + 'phone_number': '20-12-1234-1234', + 'template_id': sample_template.id + } + auth_header = create_authorization_header(service_id=sample_service.id) + + response = client.post( + path='/v2/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + assert response.status_code == 400 + assert response.headers['Content-type'] == 'application/json' + + error_json = json.loads(response.get_data(as_text=True)) + assert error_json['status_code'] == 400 + assert error_json['errors'] == [ + {"error": "BadRequestError", "message": 'Cannot send to international mobile numbers'} + ] + + +def test_post_sms_notification_returns_201_if_allowed_to_send_int_sms(notify_db, notify_db_session, client, mocker): + + service = sample_service(notify_db, notify_db_session, can_send_international_sms=True) + template = create_sample_template(notify_db, notify_db_session, service=service) + + mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + + data = { + 'phone_number': '20-12-1234-1234', + 'template_id': template.id + } + auth_header = create_authorization_header(service_id=service.id) + + response = client.post( + path='/v2/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + print(json.loads(response.get_data(as_text=True))) + assert response.status_code == 201 + assert response.headers['Content-type'] == 'application/json' + + +def test_post_sms_should_persist_supplied_sms_number(notify_api, sample_template_with_placeholders, mocker): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + data = { + 'phone_number': '+(44) 77009-00855', + 'template_id': str(sample_template_with_placeholders.id), + 'personalisation': {' Name': 'Jo'} + } + + auth_header = create_authorization_header(service_id=sample_template_with_placeholders.service_id) + + response = client.post( + path='/v2/notifications/sms', + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + assert response.status_code == 201 + resp_json = json.loads(response.get_data(as_text=True)) + notifications = Notification.query.all() + assert len(notifications) == 1 + notification_id = notifications[0].id + assert '+(44) 77009-00855' == notifications[0].to + assert resp_json['id'] == str(notification_id) + assert mocked.called diff --git a/tests/app/v2/test_errors.py b/tests/app/v2/test_errors.py index 5f0ed0f81..8345faacc 100644 --- a/tests/app/v2/test_errors.py +++ b/tests/app/v2/test_errors.py @@ -88,7 +88,6 @@ def test_validation_error(app_for_test): response = client.get(url_for('v2_under_test.raising_validation_error')) assert response.status_code == 400 error = json.loads(response.get_data(as_text=True)) - print(error) assert len(error.keys()) == 2 assert error['status_code'] == 400 assert len(error['errors']) == 2 diff --git a/tests/conftest.py b/tests/conftest.py index c7ec7ee3b..b08059725 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,7 @@ def notify_db_session(notify_db): "job_status", "provider_details_history", "template_process_type", - "dvla_organisation", - "rates"]: + "dvla_organisation"]: notify_db.engine.execute(tbl.delete()) notify_db.session.commit()