diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 5614c361f..bf10ceec7 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -8,7 +8,7 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from app import notify_celery -from app.performance_platform import total_sent_notifications +from app.performance_platform import total_sent_notifications, processing_time from app import performance_platform_client from app.dao.date_util import get_month_start_and_end_date_in_utc from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago @@ -176,27 +176,32 @@ def timeout_notifications(): @statsd(namespace="tasks") def send_daily_performance_platform_stats(): if performance_platform_client.active: - count_dict = total_sent_notifications.get_total_sent_notifications_yesterday() - email_sent_count = count_dict.get('email').get('count') - sms_sent_count = count_dict.get('sms').get('count') - start_date = count_dict.get('start_date') + send_total_sent_notifications_to_performance_platform() + processing_time.send_processing_time_to_performance_platform() - current_app.logger.info( - "Attempting to update performance platform for date {} with email count {} and sms count {}" - .format(start_date, email_sent_count, sms_sent_count) - ) - total_sent_notifications.send_total_notifications_sent_for_day_stats( - start_date, - 'sms', - sms_sent_count - ) +def send_total_sent_notifications_to_performance_platform(): + count_dict = total_sent_notifications.get_total_sent_notifications_yesterday() + email_sent_count = count_dict.get('email').get('count') + sms_sent_count = count_dict.get('sms').get('count') + start_date = count_dict.get('start_date') - total_sent_notifications.send_total_notifications_sent_for_day_stats( - start_date, - 'email', - email_sent_count - ) + current_app.logger.info( + "Attempting to update performance platform for date {} with email count {} and sms count {}" + .format(start_date, email_sent_count, sms_sent_count) + ) + + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'sms', + sms_sent_count + ) + + total_sent_notifications.send_total_notifications_sent_for_day_stats( + start_date, + 'email', + email_sent_count + ) @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery') diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 008b0fd5e..e621350ce 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -86,9 +86,11 @@ def process_job(job_id): process_row(row_number, recipient, personalisation, template, job, service) if template.template_type == LETTER_TYPE: - build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) - # temporary logging - current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id)) + if service.research_mode: + update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE) + else: + build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) + current_app.logger.info("send job {} to build-dvla-file in the {} queue".format(job_id, QueueNames.JOBS)) else: job.job_status = JOB_STATUS_FINISHED diff --git a/app/commands.py b/app/commands.py index 051cd7fc8..a27f08d1b 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1,7 +1,7 @@ import uuid -from datetime import datetime +from datetime import datetime, timedelta from decimal import Decimal -from flask_script import Command, Manager, Option +from flask_script import Command, Option from app import db from app.dao.monthly_billing_dao import ( @@ -16,6 +16,8 @@ from app.dao.services_dao import ( ) from app.dao.provider_rates_dao import create_provider_rates from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) +from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc +from app.performance_platform.processing_time import send_processing_time_for_start_and_end class CreateProviderRateCommand(Command): @@ -49,7 +51,7 @@ class PurgeFunctionalTestDataCommand(Command): Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix."), ) - def run(self, service_name_prefix=None, user_email_prefix=None): + def run(self, user_email_prefix=None): if user_email_prefix: users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all() for usr in users: @@ -167,33 +169,62 @@ class CustomDbScript(Command): class PopulateMonthlyBilling(Command): - option_list = ( - Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"), + option_list = ( + Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"), + ) + + def run(self, year): + service_ids = get_service_ids_that_need_billing_populated( + start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) ) + start, end = 1, 13 + if year == '2016': + start = 4 - def run(self, year): - service_ids = get_service_ids_that_need_billing_populated( - start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) - ) - start, end = 1, 13 - if year == '2016': - start = 4 + for service_id in service_ids: + print('Starting to populate data for service {}'.format(str(service_id))) + print('Starting populating monthly billing for {}'.format(year)) + for i in range(start, end): + print('Population for {}-{}'.format(i, year)) + self.populate(service_id, year, i) - for service_id in service_ids: - print('Starting to populate data for service {}'.format(str(service_id))) - print('Starting populating monthly billing for {}'.format(year)) - for i in range(start, end): - print('Population for {}-{}'.format(i, year)) - self.populate(service_id, year, i) + def populate(self, service_id, year, month): + create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1)) + sms_res = get_monthly_billing_by_notification_type( + service_id, datetime(int(year), int(month), 1), SMS_TYPE + ) + email_res = get_monthly_billing_by_notification_type( + service_id, datetime(int(year), int(month), 1), EMAIL_TYPE + ) + print("Finished populating data for {} for service id {}".format(month, str(service_id))) + print('SMS: {}'.format(sms_res.monthly_totals)) + print('Email: {}'.format(email_res.monthly_totals)) - def populate(self, service_id, year, month): - create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1)) - sms_res = get_monthly_billing_by_notification_type( - service_id, datetime(int(year), int(month), 1), SMS_TYPE - ) - email_res = get_monthly_billing_by_notification_type( - service_id, datetime(int(year), int(month), 1), EMAIL_TYPE - ) - print("Finished populating data for {} for service id {}".format(month, str(service_id))) - print('SMS: {}'.format(sms_res.monthly_totals)) - print('Email: {}'.format(email_res.monthly_totals)) + +class BackfillProcessingTime(Command): + option_list = ( + Option('-s', '--start_date', dest='start_date', help="Date (%Y-%m-%d) start date inclusive"), + Option('-e', '--end_date', dest='end_date', help="Date (%Y-%m-%d) end date inclusive"), + ) + + def run(self, start_date, end_date): + start_date = datetime.strptime(start_date, '%Y-%m-%d') + end_date = datetime.strptime(end_date, '%Y-%m-%d') + + delta = end_date - start_date + + print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date)) + + for i in range(delta.days + 1): + # because the tz conversion funcs talk about midnight, and the midnight before last, + # we want to pretend we're running this from the next morning, so add one. + process_date = start_date + timedelta(days=i + 1) + + process_start_date = get_midnight_for_day_before(process_date) + process_end_date = get_london_midnight_in_utc(process_date) + + print('Sending notification processing-time for {} - {}'.format( + process_start_date.isoformat(), + process_end_date.isoformat() + )) + send_processing_time_for_start_and_end(process_start_date, process_end_date) diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 4efef0e7d..606c02cc9 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -1,7 +1,5 @@ from datetime import datetime -from sqlalchemy import func - from app import db from app.dao.dao_utils import transactional diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 662857f5a..7542fb0df 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -1,4 +1,3 @@ -from collections import namedtuple from datetime import datetime, timedelta from sqlalchemy import Float, Integer @@ -17,41 +16,7 @@ from app.models import ( EMAIL_TYPE ) from app.statsd_decorators import statsd -from app.utils import get_london_month_from_utc_column, convert_utc_to_bst - - -@statsd(namespace="dao") -def get_yearly_billing_data(service_id, year): - start_date, end_date = get_financial_year(year) - rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) - - if not rates: - return [] - - def get_valid_from(valid_from): - return start_date if valid_from < start_date else valid_from - - result = [] - for r, n in zip(rates, rates[1:]): - result.append( - sms_yearly_billing_data_query( - r.rate, - service_id, - get_valid_from(r.valid_from), - n.valid_from - ) - ) - result.append( - sms_yearly_billing_data_query( - rates[-1].rate, - service_id, - get_valid_from(rates[-1].valid_from), - end_date - ) - ) - - result.append(email_yearly_billing_data_query(service_id, start_date, end_date)) - return sum(result, []) +from app.utils import get_london_month_from_utc_column @statsd(namespace="dao") @@ -112,52 +77,6 @@ def billing_data_filter(notification_type, start_date, end_date, service_id): ] -def email_yearly_billing_data_query(service_id, start_date, end_date, rate=0): - result = db.session.query( - func.count(NotificationHistory.id), - func.count(NotificationHistory.id), - rate_multiplier(), - NotificationHistory.notification_type, - NotificationHistory.international, - cast(rate, Integer()) - ).filter( - *billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - rate_multiplier(), - NotificationHistory.international - ).first() - - if not result: - return [(0, 0, 1, EMAIL_TYPE, False, 0)] - else: - return [result] - - -def sms_yearly_billing_data_query(rate, service_id, start_date, end_date): - result = db.session.query( - cast(func.sum(NotificationHistory.billable_units * rate_multiplier()), Integer()), - func.sum(NotificationHistory.billable_units), - rate_multiplier(), - NotificationHistory.notification_type, - NotificationHistory.international, - cast(rate, Float()) - ).filter( - *billing_data_filter(SMS_TYPE, start_date, end_date, service_id) - ).group_by( - NotificationHistory.notification_type, - NotificationHistory.international, - rate_multiplier() - ).order_by( - rate_multiplier() - ).all() - - if not result: - return [(0, 0, 1, SMS_TYPE, False, rate)] - else: - return result - - def get_rates_for_daterange(start_date, end_date, notification_type): rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all() diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 4514d4310..17e388422 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -16,6 +16,8 @@ from notifications_utils.recipients import ( from werkzeug.datastructures import MultiDict from sqlalchemy import (desc, func, or_, and_, asc) from sqlalchemy.orm import joinedload +from sqlalchemy.sql.expression import case +from sqlalchemy.sql import functions from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES from app import db, create_uuid @@ -42,7 +44,6 @@ from app.models import ( from app.dao.dao_utils import transactional from app.statsd_decorators import statsd -from app.utils import get_london_month_from_utc_column def dao_get_notification_statistics_for_service_and_day(service_id, day): @@ -519,3 +520,38 @@ def set_scheduled_notification_to_processed(notification_id): {'pending': False} ) db.session.commit() + + +def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date): + """ + SELECT + count(notification_history), + coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0) + FROM notification_history + WHERE + created_at > 'START DATE' AND + created_at < 'END DATE' AND + api_key_id IS NOT NULL AND + key_type != 'test' AND + notification_type != 'letter'; + """ + under_10_secs = NotificationHistory.sent_at - NotificationHistory.created_at <= timedelta(seconds=10) + sum_column = functions.coalesce(functions.sum( + case( + [ + (under_10_secs, 1) + ], + else_=0 + ) + ), 0) + + return db.session.query( + func.count(NotificationHistory.id).label('messages_total'), + sum_column.label('messages_within_10_secs') + ).filter( + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.api_key_id.isnot(None), + NotificationHistory.key_type != KEY_TYPE_TEST, + NotificationHistory.notification_type != LETTER_TYPE + ).one() diff --git a/app/dao/service_email_reply_to_dao.py b/app/dao/service_email_reply_to_dao.py new file mode 100644 index 000000000..5aaf9ee81 --- /dev/null +++ b/app/dao/service_email_reply_to_dao.py @@ -0,0 +1,32 @@ +from app import db +from app.dao.dao_utils import transactional +from app.models import ServiceEmailReplyTo + + +def create_or_update_email_reply_to(service_id, email_address): + reply_to = dao_get_reply_to_by_service_id(service_id) + if reply_to: + reply_to.email_address = email_address + dao_update_reply_to_email(reply_to) + else: + reply_to = ServiceEmailReplyTo(service_id=service_id, email_address=email_address) + dao_create_reply_to_email_address(reply_to) + + +@transactional +def dao_create_reply_to_email_address(reply_to_email): + db.session.add(reply_to_email) + + +def dao_get_reply_to_by_service_id(service_id): + reply_to = db.session.query( + ServiceEmailReplyTo + ).filter( + ServiceEmailReplyTo.service_id == service_id + ).first() + return reply_to + + +@transactional +def dao_update_reply_to_email(reply_to): + db.session.add(reply_to) diff --git a/app/job/rest.py b/app/job/rest.py index 6a8a86ee4..8b2165760 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -32,7 +32,7 @@ from app.schemas import ( from app.celery.tasks import process_job -from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED +from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED, LETTER_TYPE from app.utils import pagination_links @@ -190,6 +190,9 @@ def create_job(service_id): }) template = dao_get_template_by_id(data['template']) + if template.template_type == LETTER_TYPE and service.restricted: + raise InvalidRequest("Create letter job is not allowed for service in trial mode ", 403) + errors = unarchived_template_schema.validate({'archived': template.archived}) if errors: diff --git a/app/models.py b/app/models.py index e32456f68..588c95d48 100644 --- a/app/models.py +++ b/app/models.py @@ -245,7 +245,7 @@ class Service(db.Model, Versioned): if self.inbound_number and self.inbound_number.active: return self.inbound_number.number else: - return self.sms_sender or current_app.config['FROM_NUMBER'] + return self.sms_sender class InboundNumber(db.Model): @@ -278,6 +278,21 @@ class InboundNumber(db.Model): } +class ServiceSmsSender(db.Model): + __tablename__ = "service_sms_senders" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + sms_sender = db.Column(db.String(11), nullable=False) + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=True, index=True, nullable=False) + service = db.relationship(Service, backref=db.backref("service_sms_senders", uselist=False)) + is_default = db.Column(db.Boolean, nullable=False, default=True) + inbound_number_id = db.Column(UUID(as_uuid=True), db.ForeignKey('inbound_numbers.id'), + unique=True, index=True, nullable=True) + inbound_number = db.relationship(InboundNumber, backref=db.backref("inbound_number", uselist=False)) + created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow, nullable=False) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) + + class ServicePermission(db.Model): __tablename__ = "service_permissions" @@ -318,7 +333,7 @@ class ServiceWhitelist(db.Model): try: if recipient_type == MOBILE_TYPE: - validate_phone_number(recipient) + validate_phone_number(recipient, international=True) instance.recipient = recipient elif recipient_type == EMAIL_TYPE: validate_email_address(recipient) @@ -1314,3 +1329,17 @@ class MonthlyBilling(db.Model): def __repr__(self): return str(self.serialized()) + + +class ServiceEmailReplyTo(db.Model): + __tablename__ = "service_email_reply_to" + + id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + + service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False) + service = db.relationship(Service, backref=db.backref("reply_to_email_addresses", uselist=False)) + + email_address = db.Column(db.Text, nullable=False, index=False, unique=False) + is_default = db.Column(db.Boolean, nullable=False, default=True) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) diff --git a/app/performance_platform/processing_time.py b/app/performance_platform/processing_time.py new file mode 100644 index 000000000..b901bde84 --- /dev/null +++ b/app/performance_platform/processing_time.py @@ -0,0 +1,40 @@ +from datetime import datetime + +from flask import current_app + +from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc +from app.dao.notifications_dao import dao_get_total_notifications_sent_per_day_for_performance_platform +from app import performance_platform_client + + +def send_processing_time_to_performance_platform(): + today = datetime.utcnow() + start_date = get_midnight_for_day_before(today) + end_date = get_london_midnight_in_utc(today) + + send_processing_time_for_start_and_end(start_date, end_date) + + +def send_processing_time_for_start_and_end(start_date, end_date): + result = dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date) + + current_app.logger.info( + 'Sending processing-time to performance platform for date {}. Total: {}, under 10 secs {}'.format( + start_date, result.messages_total, result.messages_within_10_secs + ) + ) + + send_processing_time_data(start_date, 'messages-total', result.messages_total) + send_processing_time_data(start_date, 'messages-within-10-secs', result.messages_within_10_secs) + + +def send_processing_time_data(date, status, count): + payload = performance_platform_client.format_payload( + dataset='processing-time', + date=date, + group_name='status', + group_value=status, + count=count + ) + + performance_platform_client.send_stats_to_performance_platform(payload) diff --git a/app/schemas.py b/app/schemas.py index af76eef6e..5f81821d4 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -130,7 +130,7 @@ class UserUpdateAttributeSchema(BaseSchema): @validates('mobile_number') def validate_mobile_number(self, value): try: - validate_phone_number(value) + validate_phone_number(value, international=True) except InvalidPhoneError as error: raise ValidationError('Invalid phone number: {}'.format(error)) diff --git a/app/service/rest.py b/app/service/rest.py index c9f07d20f..98058774f 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,4 @@ import itertools -import json from datetime import datetime from flask import ( @@ -11,7 +10,7 @@ from flask import ( from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm.exc import NoResultFound -from app.dao import notification_usage_dao, notifications_dao +from app.dao import notifications_dao from app.dao.dao_utils import dao_rollback from app.dao.api_key_dao import ( save_model_api_key, @@ -46,6 +45,7 @@ from app.dao.service_whitelist_dao import ( dao_add_and_commit_whitelisted_contacts, dao_remove_service_whitelist ) +from app.dao.service_email_reply_to_dao import create_or_update_email_reply_to from app.dao.provider_statistics_dao import get_fragment_count from app.dao.users_dao import get_user_by_id from app.errors import ( @@ -132,9 +132,13 @@ def create_service(): @service_blueprint.route('/', methods=['POST']) def update_service(service_id): + req_json = request.get_json() fetched_service = dao_fetch_service_by_id(service_id) # Capture the status change here as Marshmallow changes this later - service_going_live = fetched_service.restricted and not request.get_json().get('restricted', True) + service_going_live = fetched_service.restricted and not req_json.get('restricted', True) + + if 'reply_to_email_address' in req_json: + create_or_update_email_reply_to(fetched_service.id, req_json['reply_to_email_address']) current_data = dict(service_schema.dump(fetched_service).data.items()) current_data.update(request.get_json()) @@ -458,43 +462,6 @@ def get_monthly_template_stats(service_id): raise InvalidRequest('Year must be a number', status_code=400) -@service_blueprint.route('//yearly-usage') -def get_yearly_billing_usage(service_id): - try: - year = int(request.args.get('year')) - results = notification_usage_dao.get_yearly_billing_data(service_id, year) - json_result = [{ - "credits": x[0], - "billing_units": x[1], - "rate_multiplier": x[2], - "notification_type": x[3], - "international": x[4], - "rate": x[5] - } for x in results] - return json.dumps(json_result) - - except TypeError: - return jsonify(result='error', message='No valid year provided'), 400 - - -@service_blueprint.route('//monthly-usage') -def get_yearly_monthly_usage(service_id): - try: - year = int(request.args.get('year')) - results = notification_usage_dao.get_monthly_billing_data(service_id, year) - json_results = [{ - "month": x[0], - "billing_units": x[1], - "rate_multiplier": x[2], - "international": x[3], - "notification_type": x[4], - "rate": x[5] - } for x in results] - return json.dumps(json_results) - except TypeError: - return jsonify(result='error', message='No valid year provided'), 400 - - @service_blueprint.route('//inbound-api', methods=['POST']) def create_service_inbound_api(service_id): data = request.get_json() diff --git a/app/utils.py b/app/utils.py index cf4c28f06..49f6d61dc 100644 --- a/app/utils.py +++ b/app/utils.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta import pytz from flask import url_for from sqlalchemy import func -from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate, LetterPreviewTemplate +from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate local_timezone = pytz.timezone("Europe/London") diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index c27ca8dc5..db69a5758 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -150,15 +150,16 @@ def process_letter_notification(*, letter_data, api_key, template): if api_key.key_type == KEY_TYPE_TEAM: raise BadRequestError(message='Cannot send letters with a team api key', status_code=403) + if api_key.service.restricted and api_key.key_type != KEY_TYPE_TEST: + raise BadRequestError(message='Cannot send letters when service is in trial mode', status_code=403) + job = create_letter_api_job(template) notification = create_letter_notification(letter_data, job, api_key) if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST: - # distinguish real API jobs from test jobs by giving the test jobs a different filename job.original_file_name = LETTER_TEST_API_FILENAME dao_update_job(job) - update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE) else: build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) diff --git a/application.py b/application.py index 0cb5377d8..3127f52af 100644 --- a/application.py +++ b/application.py @@ -17,6 +17,7 @@ manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) manager.add_command('custom_db_script', commands.CustomDbScript) manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling) +manager.add_command('backfill_processing_time', commands.BackfillProcessingTime) @manager.command diff --git a/migrations/script.py.mako b/migrations/script.py.mako index 95702017e..e17366786 100644 --- a/migrations/script.py.mako +++ b/migrations/script.py.mako @@ -1,19 +1,18 @@ -"""${message} +""" Revision ID: ${up_revision} Revises: ${down_revision} Create Date: ${create_date} """ - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} - from alembic import op import sqlalchemy as sa ${imports if imports else ""} +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + + def upgrade(): ${upgrades if upgrades else "pass"} diff --git a/migrations/versions/0117_international_sms_notify.py b/migrations/versions/0117_international_sms_notify.py new file mode 100644 index 000000000..5168e1f9c --- /dev/null +++ b/migrations/versions/0117_international_sms_notify.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: 0117_international_sms_notify +Revises: 0116_another_letter_org +Create Date: 2017-08-29 14:09:41.042061 + +""" + +# revision identifiers, used by Alembic. +revision = '0117_international_sms_notify' +down_revision = '0116_another_letter_org' + +from alembic import op +from datetime import datetime + + +NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' + + +def upgrade(): + op.execute(""" + INSERT INTO service_permissions VALUES + ('{}', 'international_sms', '{}') + """.format(NOTIFY_SERVICE_ID, datetime.utcnow())) + + +def downgrade(): + op.execute(""" + DELETE FROM service_permissions + WHERE + service_id = '{}' AND + permission = 'international_sms' + """.format(NOTIFY_SERVICE_ID)) diff --git a/migrations/versions/0118_service_sms_senders.py b/migrations/versions/0118_service_sms_senders.py new file mode 100644 index 000000000..db0c1724f --- /dev/null +++ b/migrations/versions/0118_service_sms_senders.py @@ -0,0 +1,39 @@ +"""empty message + +Revision ID: 0118_service_sms_senders +Revises: 0117_international_sms_notify +Create Date: 2017-09-05 17:29:38.921045 + +""" + +# revision identifiers, used by Alembic. +revision = '0118_service_sms_senders' +down_revision = '0117_international_sms_notify' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + op.create_table('service_sms_senders', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('sms_sender', sa.String(length=11), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('is_default', sa.Boolean(), nullable=False), + sa.Column('inbound_number_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['inbound_number_id'], ['inbound_numbers.id'], ), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_service_sms_senders_inbound_number_id'), 'service_sms_senders', ['inbound_number_id'], + unique=True) + op.create_index(op.f('ix_service_sms_senders_service_id'), 'service_sms_senders', ['service_id'], unique=True) + + +def downgrade(): + op.drop_index(op.f('ix_service_sms_senders_service_id'), table_name='service_sms_senders') + op.drop_index(op.f('ix_service_sms_senders_inbound_number_id'), table_name='service_sms_senders') + op.drop_table('service_sms_senders') diff --git a/migrations/versions/0119_add_email_reply_to.py b/migrations/versions/0119_add_email_reply_to.py new file mode 100644 index 000000000..1b5ba0a52 --- /dev/null +++ b/migrations/versions/0119_add_email_reply_to.py @@ -0,0 +1,34 @@ +""" + +Revision ID: 0119_add_email_reply_to +Revises: 0118_service_sms_senders +Create Date: 2017-09-07 15:29:49.087143 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0119_add_email_reply_to' +down_revision = '0118_service_sms_senders' + + +def upgrade(): + op.create_table('service_email_reply_to', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('email_address', sa.Text(), nullable=False), + sa.Column('is_default', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index( + op.f('ix_service_email_reply_to_service_id'), 'service_email_reply_to', ['service_id'], unique=False + ) + + +def downgrade(): + op.drop_index(op.f('ix_service_email_reply_to_service_id'), table_name='service_email_reply_to') + op.drop_table('service_email_reply_to') diff --git a/requirements.txt b/requirements.txt index 09527005b..05ac008e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -boto3==1.4.6 +boto3==1.4.7 celery==3.1.25 # pyup: <4 docopt==0.6.2 # ignore flask-bcrypt - when upgrading, it installs an invalid version of bcrypt that isn't removed when a different @@ -6,7 +6,7 @@ docopt==0.6.2 # TODO: Upgrade flask-bcrypt in a safe way Flask-Bcrypt==0.6.2 # pyup: ignore Flask-Marshmallow==0.8.0 -Flask-Migrate==2.1.0 +Flask-Migrate==2.1.1 Flask-Script==2.0.5 Flask-SQLAlchemy==2.2 Flask==0.12.2 @@ -16,19 +16,19 @@ jsonschema==2.6.0 marshmallow-sqlalchemy==0.13.1 marshmallow==2.13.6 monotonic==1.3 -psycopg2==2.7.3 +psycopg2==2.7.3.1 PyJWT==1.5.2 six==1.10.0 SQLAlchemy-Utils==0.32.14 -SQLAlchemy==1.1.13 +SQLAlchemy==1.1.14 statsd==3.2.1 -notifications-python-client==4.3.1 +notifications-python-client==4.4.0 # PaaS awscli>=1.11,<1.12 awscli-cwlogs>=1.4,<1.5 -git+https://github.com/alphagov/notifications-utils.git@20.0.3#egg=notifications-utils==20.0.3 +git+https://github.com/alphagov/notifications-utils.git@21.0.0#egg=notifications-utils==21.0.0 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index b6567ea5b..8bfd763e2 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,11 +1,11 @@ -r requirements.txt pycodestyle==2.3.1 -pytest==3.2.1 +pytest==3.2.2 pytest-mock==1.6.2 pytest-cov==2.5.1 pytest-xdist==1.20.0 coveralls==1.2.0 -moto==1.1.1 +moto==1.1.2 freezegun==0.3.9 requests-mock==1.3.0 strict-rfc3339==0.7 diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index b7bed184f..90205fd73 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -26,7 +26,7 @@ function configure_aws_logs { state_file = /home/vcap/logs/awslogs-state [/home/vcap/logs/app.log] -file = /home/vcap/logs/app.log*.json +file = /home/vcap/logs/app.log.json log_group_name = paas-${CW_APP_NAME}-application log_stream_name = {hostname} EOF diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index a05ea18f9..6cfbcf57a 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,5 +1,3 @@ -import uuid - from datetime import datetime, timedelta from functools import partial from unittest.mock import call, patch, PropertyMock @@ -29,7 +27,8 @@ from app.celery.scheduled_tasks import ( switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, timeout_notifications, - populate_monthly_billing) + populate_monthly_billing, + send_total_sent_notifications_to_performance_platform) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_job_by_id @@ -43,7 +42,7 @@ from app.models import ( SMS_TYPE, LETTER_TYPE, JOB_STATUS_READY_TO_SEND, MonthlyBilling) -from app.utils import get_london_midnight_in_utc, convert_utc_to_bst +from app.utils import get_london_midnight_in_utc from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( sample_job as create_sample_job, @@ -285,7 +284,7 @@ def test_send_daily_performance_stats_calls_does_not_send_if_inactive(client, mo @freeze_time("2016-01-11 12:30:00") -def test_send_daily_performance_stats_calls_with_correct_totals( +def test_send_total_sent_notifications_to_performance_platform_calls_with_correct_totals( notify_db, notify_db_session, sample_template, @@ -319,7 +318,7 @@ def test_send_daily_performance_stats_calls_with_correct_totals( new_callable=PropertyMock ) as mock_active: mock_active.return_value = True - send_daily_performance_platform_stats() + send_total_sent_notifications_to_performance_platform() perf_mock.assert_has_calls([ call(get_london_midnight_in_utc(yesterday), 'sms', 2), @@ -682,7 +681,7 @@ def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template): def test_run_letter_jobs(client, mocker, sample_letter_template): jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND), create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)] - job_ids = [str(job.id) for job in jobs] + job_ids = [str(j.id) for j in jobs] mocker.patch( "app.celery.scheduled_tasks.dao_get_letter_job_ids_by_status", return_value=job_ids diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 717dc7b82..a12e6c050 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -29,6 +29,7 @@ from app.celery.tasks import ( update_letter_notifications_statuses, process_updates_from_file, send_inbound_sms_to_service) +from app.config import QueueNames from app.dao import jobs_dao, services_dao from app.models import ( Notification, @@ -38,7 +39,8 @@ from app.models import ( SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, - Job) + Job, + JOB_STATUS_ERROR) from tests.app import load_example_csv from tests.conftest import set_config @@ -608,6 +610,62 @@ def test_should_put_send_email_task_in_research_mode_queue_if_research_mode_serv ) +def test_should_not_build_dvla_file_in_research_mode_for_letter_job( + mocker, sample_service, sample_letter_job, fake_uuid +): + test_encrypted_data = 'some encrypted data' + sample_service.research_mode = True + + csv = """address_line_1,address_line_2,address_line_3,address_line_4,postcode,name + A1,A2,A3,A4,A_POST,Alice + """ + mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=csv) + mocker.patch('app.celery.tasks.update_job_to_sent_to_dvla.apply_async') + mocker.patch('app.celery.tasks.persist_letter.apply_async') + mocker.patch('app.celery.tasks.create_uuid', return_value=fake_uuid) + mocker.patch('app.celery.tasks.encryption.encrypt', return_value=test_encrypted_data) + mock_dvla_file_task = mocker.patch('app.celery.tasks.build_dvla_file.apply_async') + + process_job(sample_letter_job.id) + + assert not mock_dvla_file_task.called + + +@freeze_time("2017-08-29 17:30:00") +def test_should_update_job_to_sent_to_dvla_in_research_mode_for_letter_job( + mocker, sample_service, sample_letter_job, fake_uuid +): + test_encrypted_data = 'some encrypted data' + sample_service.research_mode = True + + csv = """address_line_1,address_line_2,address_line_3,address_line_4,postcode,name + A1,A2,A3,A4,A_POST,Alice + """ + mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=csv) + mocker.patch('app.celery.tasks.update_job_to_sent_to_dvla.apply_async') + mocker.patch('app.celery.tasks.persist_letter.apply_async') + mocker.patch('app.celery.tasks.create_uuid', return_value=fake_uuid) + mocker.patch('app.celery.tasks.encryption.encrypt', return_value=test_encrypted_data) + mock_dvla_file_task = mocker.patch('app.celery.tasks.build_dvla_file.apply_async') + + process_job(sample_letter_job.id) + + job = jobs_dao.dao_get_job_by_id(sample_letter_job.id) + + persist_letter.apply_async.assert_called_once_with( + ( + str(sample_service.id), + fake_uuid, + test_encrypted_data, + datetime.utcnow().strftime(DATETIME_FORMAT) + ), + queue=QueueNames.RESEARCH_MODE + ) + + update_job_to_sent_to_dvla.apply_async.assert_called_once_with( + [str(job.id)], queue=QueueNames.RESEARCH_MODE) + + def test_should_send_sms_template_to_and_persist_with_job_id(sample_job, sample_api_key, mocker): notification = _notification_json( sample_job.template, diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 86437e7a1..f23bc6200 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -136,24 +136,30 @@ def sample_service( restricted=False, limit=1000, email_from=None, - permissions=[SMS_TYPE, EMAIL_TYPE] + permissions=[SMS_TYPE, EMAIL_TYPE], + research_mode=None ): if user is None: user = create_user() if email_from is None: email_from = service_name.lower().replace(' ', '.') + data = { 'name': service_name, 'message_limit': limit, 'restricted': restricted, 'email_from': email_from, 'created_by': user, - 'letter_contact_block': 'London,\nSW1A 1AA', + 'letter_contact_block': 'London,\nSW1A 1AA' } service = Service.query.filter_by(name=service_name).first() if not service: service = Service(**data) dao_create_service(service, user, service_permissions=permissions) + + if research_mode: + service.research_mode = research_mode + else: if user not in service.users: dao_add_user_to_service(service, user) @@ -206,6 +212,7 @@ def sample_template( }) template = Template(**data) dao_create_template(template) + return template @@ -271,6 +278,12 @@ def sample_letter_template(sample_service_full_permissions): return create_template(sample_service_full_permissions, template_type=LETTER_TYPE) +@pytest.fixture +def sample_trial_letter_template(sample_service_full_permissions): + sample_service_full_permissions.restricted = True + return create_template(sample_service_full_permissions, template_type=LETTER_TYPE) + + @pytest.fixture(scope='function') def sample_email_template_with_placeholders(notify_db, notify_db_session): return sample_email_template( @@ -631,14 +644,22 @@ def sample_notification_history( status='created', created_at=None, notification_type=None, - key_type=KEY_TYPE_NORMAL + key_type=KEY_TYPE_NORMAL, + sent_at=None, + api_key=None ): if created_at is None: created_at = datetime.utcnow() + if sent_at is None: + sent_at = datetime.utcnow() + if notification_type is None: notification_type = sample_template.template_type + if not api_key: + api_key = create_api_key(sample_template.service, key_type=key_type) + notification_history = NotificationHistory( id=uuid.uuid4(), service=sample_template.service, @@ -647,7 +668,10 @@ def sample_notification_history( status=status, created_at=created_at, notification_type=notification_type, - key_type=key_type + key_type=key_type, + api_key=api_key, + api_key_id=api_key and api_key.id, + sent_at=sent_at ) notify_db.session.add(notification_history) notify_db.session.commit() diff --git a/tests/app/dao/notification_dao/__init__.py b/tests/app/dao/notification_dao/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py similarity index 100% rename from tests/app/dao/test_notification_dao.py rename to tests/app/dao/notification_dao/test_notification_dao.py diff --git a/tests/app/dao/notification_dao/test_notification_dao_performance_platform.py b/tests/app/dao/notification_dao/test_notification_dao_performance_platform.py new file mode 100644 index 000000000..125223e2a --- /dev/null +++ b/tests/app/dao/notification_dao/test_notification_dao_performance_platform.py @@ -0,0 +1,135 @@ +from datetime import date, datetime, timedelta + +from freezegun import freeze_time + +from app.dao.notifications_dao import dao_get_total_notifications_sent_per_day_for_performance_platform +from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST + +from tests.app.db import create_notification +from tests.app.conftest import ( + sample_notification_history, + sample_service, + sample_template +) + +BEGINNING_OF_DAY = date(2016, 10, 18) +END_OF_DAY = date(2016, 10, 19) + + +def test_get_total_notifications_filters_on_date_within_date_range(sample_template): + create_notification(sample_template, created_at=datetime(2016, 10, 17, 23, 59, 59)) + create_notification(sample_template, created_at=BEGINNING_OF_DAY) + create_notification(sample_template, created_at=datetime(2016, 10, 18, 23, 59, 59)) + create_notification(sample_template, created_at=END_OF_DAY) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 2 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_only_counts_api_notifications(sample_template, sample_job, sample_api_key): + create_notification(sample_template, one_off=True) + create_notification(sample_template, one_off=True) + create_notification(sample_template, job=sample_job) + create_notification(sample_template, job=sample_job) + create_notification(sample_template, api_key=sample_api_key) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 1 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_ignores_test_keys(sample_template): + # Creating multiple templates with normal and team keys but only 1 template + # with a test key to test that the count ignores letters + create_notification(sample_template, key_type=KEY_TYPE_NORMAL) + create_notification(sample_template, key_type=KEY_TYPE_NORMAL) + create_notification(sample_template, key_type=KEY_TYPE_TEAM) + create_notification(sample_template, key_type=KEY_TYPE_TEAM) + create_notification(sample_template, key_type=KEY_TYPE_TEST) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 4 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_ignores_letters( + sample_template, + sample_email_template, + sample_letter_template +): + # Creating multiple sms and email templates but only 1 letter template to + # test that the count ignores letters + create_notification(sample_template) + create_notification(sample_template) + create_notification(sample_email_template) + create_notification(sample_email_template) + create_notification(sample_letter_template) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 4 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_counts_messages_within_10_seconds(sample_template): + created_at = datetime.utcnow() + + create_notification(sample_template, sent_at=created_at + timedelta(seconds=5)) + create_notification(sample_template, sent_at=created_at + timedelta(seconds=10)) + create_notification(sample_template, sent_at=created_at + timedelta(seconds=15)) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 3 + assert result.messages_within_10_secs == 2 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_counts_messages_that_have_not_sent(sample_template): + create_notification(sample_template, status='created', sent_at=None) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 1 + assert result.messages_within_10_secs == 0 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_returns_zero_if_no_data(notify_db_session): + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 0 + assert result.messages_within_10_secs == 0 + + +@freeze_time('2016-10-18T10:00') +def test_get_total_notifications_counts_ignores_research_mode(notify_db, notify_db_session): + created_at = datetime.utcnow() + service = sample_service(notify_db, notify_db_session, research_mode=True) + template = sample_template(notify_db, notify_db_session, service=service) + + create_notification(template, status='created', sent_at=None) + + sample_notification_history( + notify_db, + notify_db_session, + template, + notification_type='email', + sent_at=created_at + timedelta(seconds=5) + ) + sample_notification_history( + notify_db, + notify_db_session, + template, + notification_type='sms', + sent_at=created_at + timedelta(seconds=5) + ) + + result = dao_get_total_notifications_sent_per_day_for_performance_platform(BEGINNING_OF_DAY, END_OF_DAY) + + assert result.messages_total == 2 + assert result.messages_within_10_secs == 2 diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py index 24359af3a..23c445f92 100644 --- a/tests/app/dao/test_monthly_billing.py +++ b/tests/app/dao/test_monthly_billing.py @@ -3,7 +3,6 @@ from dateutil.relativedelta import relativedelta from freezegun import freeze_time from functools import partial -from app import db from app.dao.monthly_billing_dao import ( create_or_update_monthly_billing, get_monthly_billing_entry, diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 02876e6fb..24f09e026 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -6,7 +6,6 @@ from freezegun import freeze_time from app.dao.date_util import get_financial_year from app.dao.notification_usage_dao import ( get_rates_for_daterange, - get_yearly_billing_data, get_billing_data_for_month, get_monthly_billing_data ) @@ -21,7 +20,7 @@ def test_get_rates_for_daterange(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) set_up_rate(notify_db, datetime(2017, 3, 31, 23), 0.0158) start_date, end_date = get_financial_year(2017) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates) == 1 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-03-31 23:00:00" assert rates[0].rate == 0.0158 @@ -32,7 +31,7 @@ def test_get_rates_for_daterange_multiple_result_per_year(notify_db, notify_db_s set_up_rate(notify_db, datetime(2016, 5, 18), 0.016) set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates) == 2 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" assert rates[0].rate == 0.015 @@ -45,7 +44,7 @@ def test_get_rates_for_daterange_returns_correct_rates(notify_db, notify_db_sess set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2017) - rates_2017 = get_rates_for_daterange(start_date, end_date, 'sms') + rates_2017 = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates_2017) == 2 assert datetime.strftime(rates_2017[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-09-01 00:00:00" assert rates_2017[0].rate == 0.016 @@ -57,7 +56,7 @@ def test_get_rates_for_daterange_in_the_future(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2018) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" assert rates[0].rate == 0.0175 @@ -66,7 +65,7 @@ def test_get_rates_for_daterange_returns_empty_list_if_year_is_before_earliest_r set_up_rate(notify_db, datetime(2016, 4, 1), 0.015) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2015) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert rates == [] @@ -76,7 +75,7 @@ def test_get_rates_for_daterange_early_rate(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 9, 1), 0.016) set_up_rate(notify_db, datetime(2017, 6, 1), 0.0175) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates) == 3 @@ -84,7 +83,7 @@ def test_get_rates_for_daterange_edge_case(notify_db, notify_db_session): set_up_rate(notify_db, datetime(2016, 3, 31, 23, 00), 0.015) set_up_rate(notify_db, datetime(2017, 3, 31, 23, 00), 0.0175) start_date, end_date = get_financial_year(2016) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates) == 1 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-03-31 23:00:00" assert rates[0].rate == 0.015 @@ -97,100 +96,12 @@ def test_get_rates_for_daterange_where_daterange_is_one_month_that_falls_between set_up_rate(notify_db, datetime(2017, 3, 31), 0.123) start_date = datetime(2017, 2, 1, 00, 00, 00) end_date = datetime(2017, 2, 28, 23, 59, 59, 99999) - rates = get_rates_for_daterange(start_date, end_date, 'sms') + rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) assert len(rates) == 1 assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-01-01 00:00:00" assert rates[0].rate == 0.175 -def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - set_up_rate(notify_db, datetime(2016, 6, 1), 0.0158) - set_up_rate(notify_db, datetime(2017, 6, 1), 0.0165) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3, rate_multiplier=2, international=True, phone_prefix="1") - create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), - status='sending', billable_units=4) - create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), - status='sending', billable_units=5) - create_notification(template=sample_email_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), - status='sending', billable_units=0) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), - status='sending', billable_units=6) - results = get_yearly_billing_data(sample_template.service_id, 2016) - assert len(results) == 4 - assert results[0] == (3, 3, 1, 'sms', False, 0.014) - assert results[1] == (9, 9, 1, 'sms', False, 0.0158) - assert results[2] == (6, 3, 2, 'sms', True, 0.0158) - assert results[3] == (2, 2, 1, 'email', False, 0) - - -def test_get_future_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): - set_up_rate(notify_db, datetime(2017, 4, 1), 0.0158) - - create_notification(template=sample_template, created_at=datetime(2017, 3, 30), sent_at=datetime(2017, 3, 30), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2017, 4, 6), sent_at=datetime(2017, 4, 6), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2017, 4, 6), sent_at=datetime(2017, 4, 6), - status='sending', billable_units=1) - - results = get_yearly_billing_data(sample_template.service_id, 2018) - assert len(results) == 2 - assert results[0] == (0, 0, 1, 'sms', False, 0.0158) - - -def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - # previous year - create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=1) - # current year - create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=1) - create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), - status='sending', billable_units=2) - create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), - status='sending', billable_units=3) - create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), - status='sending', billable_units=4) - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 22, 59, 59), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=5) - # next year - create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), - sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) - create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), - status='sending', billable_units=7) - results = get_yearly_billing_data(sample_template.service_id, 2016) - assert len(results) == 2 - assert results[0] == (15, 15, 1, 'sms', False, 0.014) - assert results[1] == (0, 0, 1, 'email', False, 0) - - -def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template): - set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) - create_notification(template=sample_email_template, created_at=datetime(2016, 7, 31), sent_at=datetime(2016, 3, 31), - status='sending', billable_units=0) - create_notification(template=sample_email_template, created_at=datetime(2016, 10, 2), sent_at=datetime(2016, 4, 2), - status='sending', billable_units=0) - - results = get_yearly_billing_data(sample_email_template.service_id, 2016) - assert len(results) == 2 - assert results[0] == (0, 0, 1, 'sms', False, 0.014) - assert results[1] == (2, 2, 1, 'email', False, 0) - - def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): set_up_rate(notify_db, datetime(2016, 4, 1), 0.014) # previous year @@ -220,10 +131,10 @@ def test_get_monthly_billing_data(notify_db, notify_db_session, sample_template, results = get_monthly_billing_data(sample_template.service_id, 2016) assert len(results) == 4 # (billable_units, rate_multiplier, international, type, rate) - assert results[0] == ('April', 1, 1, False, 'sms', 0.014) - assert results[1] == ('May', 2, 1, False, 'sms', 0.014) - assert results[2] == ('July', 7, 1, False, 'sms', 0.014) - assert results[3] == ('July', 6, 2, False, 'sms', 0.014) + assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) + assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) + assert results[2] == ('July', 7, 1, False, SMS_TYPE, 0.014) + assert results[3] == ('July', 6, 2, False, SMS_TYPE, 0.014) def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_session, sample_template, @@ -254,10 +165,10 @@ def test_get_monthly_billing_data_with_multiple_rates(notify_db, notify_db_sessi sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) results = get_monthly_billing_data(sample_template.service_id, 2016) assert len(results) == 4 - assert results[0] == ('April', 1, 1, False, 'sms', 0.014) - assert results[1] == ('May', 2, 1, False, 'sms', 0.014) - assert results[2] == ('June', 3, 1, False, 'sms', 0.014) - assert results[3] == ('June', 4, 1, False, 'sms', 0.0175) + assert results[0] == ('April', 1, 1, False, SMS_TYPE, 0.014) + assert results[1] == ('May', 2, 1, False, SMS_TYPE, 0.014) + assert results[2] == ('June', 3, 1, False, SMS_TYPE, 0.014) + assert results[3] == ('June', 4, 1, False, SMS_TYPE, 0.0175) def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, notify_db_session, sample_template): @@ -267,23 +178,10 @@ def test_get_monthly_billing_data_with_no_notifications_for_daterange(notify_db, def set_up_rate(notify_db, start_date, value): - rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=SMS_TYPE) notify_db.session.add(rate) -def test_get_yearly_billing_data_for_start_date_before_rate_returns_empty( - sample_template -): - create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) - - results = get_yearly_billing_data( - service_id=sample_template.service_id, - year=2015 - ) - - assert not results - - @freeze_time("2016-05-01") def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty( sample_template diff --git a/tests/app/dao/test_service_email_reply_to_dao.py b/tests/app/dao/test_service_email_reply_to_dao.py new file mode 100644 index 000000000..e1de4ff7e --- /dev/null +++ b/tests/app/dao/test_service_email_reply_to_dao.py @@ -0,0 +1,41 @@ +from app.dao.service_email_reply_to_dao import ( + create_or_update_email_reply_to, + dao_get_reply_to_by_service_id +) +from app.models import ServiceEmailReplyTo +from tests.app.db import create_reply_to_email, create_service + + +def test_create_or_update_email_reply_to_does_not_create_another_entry(notify_db_session): + service = create_service() + create_reply_to_email(service, 'test@mail.com') + + create_or_update_email_reply_to(service.id, 'different@mail.com') + + reply_to = dao_get_reply_to_by_service_id(service.id) + + assert ServiceEmailReplyTo.query.count() == 1 + + +def test_create_or_update_email_reply_to_updates_existing_entry(notify_db_session): + service = create_service() + create_reply_to_email(service, 'test@mail.com') + + create_or_update_email_reply_to(service.id, 'different@mail.com') + + reply_to = dao_get_reply_to_by_service_id(service.id) + + assert reply_to.service.id == service.id + assert reply_to.email_address == 'different@mail.com' + + +def test_create_or_update_email_reply_to_creates_new_entry(notify_db_session): + service = create_service() + + create_or_update_email_reply_to(service.id, 'test@mail.com') + + reply_to = dao_get_reply_to_by_service_id(service.id) + + assert ServiceEmailReplyTo.query.count() == 1 + assert reply_to.service.id == service.id + assert reply_to.email_address == 'test@mail.com' diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index c815eb66b..22e1670f3 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -23,19 +23,24 @@ from app.models import User, VerifyCode from tests.app.db import create_user -def test_create_user(notify_db_session): +@pytest.mark.parametrize('phone_number', [ + '+447700900986', + '+1-800-555-5555', +]) +def test_create_user(notify_db_session, phone_number): email = 'notify@digital.cabinet-office.gov.uk' data = { 'name': 'Test User', 'email_address': email, 'password': 'password', - 'mobile_number': '+447700900986' + 'mobile_number': phone_number } user = User(**data) save_model_user(user) assert User.query.count() == 1 assert User.query.first().email_address == email assert User.query.first().id == user.id + assert User.query.first().mobile_number == phone_number assert not user.platform_admin diff --git a/tests/app/db.py b/tests/app/db.py index 8283428c9..a1323be7d 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -25,7 +25,9 @@ from app.models import ( SMS_TYPE, INBOUND_SMS_TYPE, KEY_TYPE_NORMAL, - ServiceInboundApi) + ServiceInboundApi, + ServiceEmailReplyTo +) from app.dao.users_dao import save_model_user from app.dao.notifications_dao import dao_create_notification, dao_created_scheduled_notification from app.dao.templates_dao import dao_create_template @@ -123,7 +125,8 @@ def create_notification( international=False, phone_prefix=None, scheduled_for=None, - normalised_to=None + normalised_to=None, + one_off=False, ): if created_at is None: created_at = datetime.utcnow() @@ -132,7 +135,7 @@ def create_notification( sent_at = sent_at or datetime.utcnow() updated_at = updated_at or datetime.utcnow() - if job is None and api_key is None: + if not one_off and (job is None and api_key is None): # we didn't specify in test - lets create it api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first() if not api_key: @@ -326,3 +329,19 @@ def create_monthly_billing_entry( db.session.commit() return entry + + +def create_reply_to_email( + service, + email_address +): + data = { + 'service': service, + 'email_address': email_address, + } + reply_to = ServiceEmailReplyTo(**data) + + db.session.add(reply_to) + db.session.commit() + + return reply_to diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index 7ce45b51e..920bc3397 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -186,6 +186,29 @@ def test_create_job_returns_403_if_service_is_not_active(client, fake_uuid, samp mock_job_dao.assert_not_called() +def test_create_job_returns_403_if_letter_template_type_and_service_in_trial( + client, fake_uuid, sample_service, sample_trial_letter_template, mocker): + data = { + 'id': fake_uuid, + 'service': str(sample_trial_letter_template.service.id), + 'template': str(sample_trial_letter_template.id), + 'original_file_name': 'thisisatest.csv', + 'notification_count': 1, + 'created_by': str(sample_trial_letter_template.created_by.id) + } + mock_job_dao = mocker.patch("app.dao.jobs_dao.dao_create_job") + auth_header = create_authorization_header() + response = client.post('/service/{}/job'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header]) + + assert response.status_code == 403 + resp_json = json.loads(response.get_data(as_text=True)) + assert resp_json['result'] == 'error' + assert resp_json['message'] == "Create letter job is not allowed for service in trial mode " + mock_job_dao.assert_not_called() + + def test_should_not_create_scheduled_job_more_then_24_hours_hence(notify_api, sample_template, mocker, fake_uuid): with notify_api.test_request_context(): with notify_api.test_client() as client: diff --git a/tests/app/letters/test_send_letter_jobs.py b/tests/app/letters/test_send_letter_jobs.py index d38e24c8f..d899387b6 100644 --- a/tests/app/letters/test_send_letter_jobs.py +++ b/tests/app/letters/test_send_letter_jobs.py @@ -5,11 +5,15 @@ from flask import json from app.variables import LETTER_TEST_API_FILENAME from tests import create_authorization_header +from tests.app.db import create_job -def test_send_letter_jobs(client, mocker): +def test_send_letter_jobs(client, mocker, sample_letter_template): mock_celery = mocker.patch("app.letters.rest.notify_celery.send_task") - job_ids = {"job_ids": [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())]} + job_1 = create_job(sample_letter_template) + job_2 = create_job(sample_letter_template) + job_3 = create_job(sample_letter_template) + job_ids = {"job_ids": [str(job_1.id), str(job_2.id), str(job_3.id)]} auth_header = create_authorization_header() diff --git a/tests/app/performance_platform/test_processing_time.py b/tests/app/performance_platform/test_processing_time.py new file mode 100644 index 000000000..07f78859e --- /dev/null +++ b/tests/app/performance_platform/test_processing_time.py @@ -0,0 +1,47 @@ +from datetime import datetime, timedelta + +from freezegun import freeze_time + +from tests.app.db import create_notification +from app.performance_platform.processing_time import ( + send_processing_time_to_performance_platform, + send_processing_time_data +) + + +@freeze_time('2016-10-18T02:00') +def test_send_processing_time_to_performance_platform_generates_correct_calls(mocker, sample_template): + send_mock = mocker.patch('app.performance_platform.processing_time.send_processing_time_data') + + created_at = datetime.utcnow() - timedelta(days=1) + + create_notification(sample_template, created_at=created_at, sent_at=created_at + timedelta(seconds=5)) + create_notification(sample_template, created_at=created_at, sent_at=created_at + timedelta(seconds=15)) + create_notification(sample_template, created_at=datetime.utcnow() - timedelta(days=2)) + + send_processing_time_to_performance_platform() + + send_mock.assert_any_call(datetime(2016, 10, 16, 23, 0), 'messages-total', 2) + send_mock.assert_any_call(datetime(2016, 10, 16, 23, 0), 'messages-within-10-secs', 1) + + +def test_send_processing_time_to_performance_platform_creates_correct_call_to_perf_platform(mocker): + send_stats = mocker.patch('app.performance_platform.total_sent_notifications.performance_platform_client.send_stats_to_performance_platform') # noqa + + send_processing_time_data( + date=datetime(2016, 10, 15, 23, 0, 0), + status='foo', + count=142 + ) + + assert send_stats.call_count == 1 + + request_args = send_stats.call_args[0][0] + assert request_args['dataType'] == 'processing-time' + assert request_args['service'] == 'govuk-notify' + assert request_args['period'] == 'day' + assert request_args['status'] == 'foo' + assert request_args['_timestamp'] == '2016-10-16T00:00:00' + assert request_args['count'] == 142 + expected_base64_id = 'MjAxNi0xMC0xNlQwMDowMDowMGdvdnVrLW5vdGlmeWZvb3Byb2Nlc3NpbmctdGltZWRheQ==' + assert request_args['_id'] == expected_base64_id diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index af6a815dc..0d8e3fea2 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -8,12 +8,11 @@ import pytest from flask import url_for, current_app from freezegun import freeze_time -from app.dao.monthly_billing_dao import create_or_update_monthly_billing from app.dao.services_dao import dao_remove_user_from_service from app.dao.templates_dao import dao_redact_template from app.dao.users_dao import save_model_user from app.models import ( - User, Organisation, Rate, Service, ServicePermission, Notification, + User, Organisation, Service, ServicePermission, Notification, DVLA_ORG_LAND_REGISTRY, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE, @@ -1730,119 +1729,6 @@ def test_get_template_stats_by_month_returns_error_for_incorrect_year( assert json.loads(response.get_data(as_text=True)) == expected_json -def test_get_yearly_billing_usage(client, notify_db, notify_db_session, sample_service): - rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=0.0158, notification_type=SMS_TYPE) - notify_db.session.add(rate) - after_rate_created = datetime(2016, 6, 5) - notification = create_sample_notification( - notify_db, - notify_db_session, - created_at=after_rate_created, - sent_at=after_rate_created, - status='sending', - service=sample_service - ) - create_or_update_monthly_billing(sample_service.id, after_rate_created) - response = client.get( - '/service/{}/yearly-usage?year=2016'.format(notification.service_id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - - assert json.loads(response.get_data(as_text=True)) == [{'credits': 1, - 'billing_units': 1, - 'rate_multiplier': 1, - 'notification_type': SMS_TYPE, - 'international': False, - 'rate': 0.0158}, - {'credits': 0, - 'billing_units': 0, - 'rate_multiplier': 1, - 'notification_type': EMAIL_TYPE, - 'international': False, - 'rate': 0}] - - -def test_get_yearly_billing_usage_returns_400_if_missing_year(client, sample_service): - response = client.get( - '/service/{}/yearly-usage'.format(sample_service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 400 - assert json.loads(response.get_data(as_text=True)) == { - 'message': 'No valid year provided', 'result': 'error' - } - - -def test_get_monthly_billing_usage(client, notify_db, notify_db_session, sample_service): - rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=0.0158, notification_type=SMS_TYPE) - notify_db.session.add(rate) - notification = create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), - sent_at=datetime(2016, 6, 5), - status='sending') - create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), - sent_at=datetime(2016, 6, 5), - status='sending', rate_multiplier=2) - create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 7, 5), - sent_at=datetime(2016, 7, 5), - status='sending') - - template = create_template(sample_service, template_type=EMAIL_TYPE) - create_sample_notification(notify_db, notify_db_session, created_at=datetime(2016, 6, 5), - sent_at=datetime(2016, 6, 5), - status='sending', - template=template) - response = client.get( - '/service/{}/monthly-usage?year=2016'.format(notification.service_id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - actual = json.loads(response.get_data(as_text=True)) - assert len(actual) == 3 - assert actual == [{'month': 'June', - 'international': False, - 'rate_multiplier': 1, - 'notification_type': SMS_TYPE, - 'rate': 0.0158, - 'billing_units': 1}, - {'month': 'June', - 'international': False, - 'rate_multiplier': 2, - 'notification_type': SMS_TYPE, - 'rate': 0.0158, - 'billing_units': 1}, - {'month': 'July', - 'international': False, - 'rate_multiplier': 1, - 'notification_type': SMS_TYPE, - 'rate': 0.0158, - 'billing_units': 1}] - - -def test_get_monthly_billing_usage_returns_400_if_missing_year(client, sample_service): - response = client.get( - '/service/{}/monthly-usage'.format(sample_service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 400 - assert json.loads(response.get_data(as_text=True)) == { - 'message': 'No valid year provided', 'result': 'error' - } - - -def test_get_monthly_billing_usage_returns_empty_list_if_no_notifications(client, notify_db, sample_service): - rate = Rate(id=uuid.uuid4(), valid_from=datetime(2016, 3, 31, 23, 00), rate=0.0158, notification_type=SMS_TYPE) - notify_db.session.add(rate) - response = client.get( - '/service/{}/monthly-usage?year=2016'.format(sample_service.id), - headers=[create_authorization_header()] - ) - assert response.status_code == 200 - - results = json.loads(response.get_data(as_text=True)) - assert results == [] - - def test_search_for_notification_by_to_field(client, notify_db, notify_db_session): create_notification = partial(create_sample_notification, notify_db, notify_db_session) notification1 = create_notification(to_field='+447700900855', normalised_to='447700900855') @@ -2237,3 +2123,18 @@ def test_is_service_name_unique_returns_400_when_name_does_not_exist(client): json_resp = json.loads(response.get_data(as_text=True)) assert json_resp["message"][0]["name"] == ["Can't be empty"] assert json_resp["message"][1]["email_from"] == ["Can't be empty"] + + +def test_update_service_reply_to_email_address_upserts_email_reply_to(mocker, admin_request, sample_service): + update_mock = mocker.patch('app.service.rest.create_or_update_email_reply_to') + + admin_request.post( + 'service.update_service', + service_id=sample_service.id, + _data={ + 'reply_to_email_address': 'new@mail.com' + }, + _expected_status=200 + ) + + assert update_mock.called diff --git a/tests/app/service/test_service_whitelist.py b/tests/app/service/test_service_whitelist.py index 23f6a3aa2..18c0595f8 100644 --- a/tests/app/service/test_service_whitelist.py +++ b/tests/app/service/test_service_whitelist.py @@ -24,15 +24,15 @@ def test_get_whitelist_returns_data(client, sample_service_whitelist): def test_get_whitelist_separates_emails_and_phones(client, sample_service): dao_add_and_commit_whitelisted_contacts([ ServiceWhitelist.from_string(sample_service.id, EMAIL_TYPE, 'service@example.com'), - ServiceWhitelist.from_string(sample_service.id, MOBILE_TYPE, '07123456789') + ServiceWhitelist.from_string(sample_service.id, MOBILE_TYPE, '07123456789'), + ServiceWhitelist.from_string(sample_service.id, MOBILE_TYPE, '+1800-555-555'), ]) response = client.get('service/{}/whitelist'.format(sample_service.id), headers=[create_authorization_header()]) assert response.status_code == 200 - assert json.loads(response.get_data(as_text=True)) == { - 'email_addresses': ['service@example.com'], - 'phone_numbers': ['07123456789'] - } + json_resp = json.loads(response.get_data(as_text=True)) + assert json_resp['email_addresses'] == ['service@example.com'] + assert sorted(json_resp['phone_numbers']) == sorted(['+1800-555-555', '07123456789']) def test_get_whitelist_404s_with_unknown_service_id(client): diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 72b2f2865..8f90df90d 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -213,7 +213,7 @@ def test_sms_inbound_config(): def test_performance_platform_config(): extract_cloudfoundry_config() - assert os.environ['PERFORMANCE_PLATFORM_ENDPOINTS'] == json.dumps({ + assert json.loads(os.environ['PERFORMANCE_PLATFORM_ENDPOINTS']) == { 'foo': 'my_token', 'bar': 'other_token' - }) + } diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py new file mode 100644 index 000000000..1c5ceedd9 --- /dev/null +++ b/tests/app/test_commands.py @@ -0,0 +1,14 @@ +from datetime import datetime + +from app.commands import BackfillProcessingTime + + +def test_backfill_processing_time_works_for_correct_dates(mocker): + send_mock = mocker.patch('app.commands.send_processing_time_for_start_and_end') + + BackfillProcessingTime().run('2017-08-01', '2017-08-03') + + assert send_mock.call_count == 3 + send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0)) + send_mock.assert_any_call(datetime(2017, 8, 1, 23, 0), datetime(2017, 8, 2, 23, 0)) + send_mock.assert_any_call(datetime(2017, 8, 2, 23, 0), datetime(2017, 8, 3, 23, 0)) diff --git a/tests/app/v2/notifications/test_post_letter_notifications.py b/tests/app/v2/notifications/test_post_letter_notifications.py index 429e53eb8..aa4433a59 100644 --- a/tests/app/v2/notifications/test_post_letter_notifications.py +++ b/tests/app/v2/notifications/test_post_letter_notifications.py @@ -17,13 +17,12 @@ from app.variables import LETTER_TEST_API_FILENAME from app.variables import LETTER_API_FILENAME from tests import create_authorization_header -from tests.app.db import create_service -from tests.app.db import create_template +from tests.app.db import create_service, create_template def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201): resp = client.post( - url_for('v2_notifications.post_notification', notification_type='letter'), + url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE), data=json.dumps(data), headers=[ ('Content-Type', 'application/json'), @@ -232,3 +231,37 @@ def test_post_letter_notification_doesnt_accept_team_key(client, sample_letter_t assert error_json['status_code'] == 403 assert error_json['errors'] == [{'error': 'BadRequestError', 'message': 'Cannot send letters with a team api key'}] + + +def test_post_letter_notification_doesnt_send_in_trial(client, sample_trial_letter_template): + data = { + 'template_id': str(sample_trial_letter_template.id), + 'personalisation': {'address_line_1': 'Foo', 'address_line_2': 'Bar', 'postcode': 'Baz'} + } + + error_json = letter_request( + client, + data, + sample_trial_letter_template.service_id, + _expected_status=403 + ) + + assert error_json['status_code'] == 403 + assert error_json['errors'] == [ + {'error': 'BadRequestError', 'message': 'Cannot send letters when service is in trial mode'}] + + +def test_post_letter_notification_calls_update_job_sent_to_dvla_when_service_is_in_trial_mode_but_using_test_key( + client, sample_trial_letter_template, mocker): + build_dvla_task = mocker.patch('app.celery.tasks.build_dvla_file.apply_async') + update_job_task = mocker.patch('app.celery.tasks.update_job_to_sent_to_dvla.apply_async') + + data = { + "template_id": sample_trial_letter_template.id, + "personalisation": {'address_line_1': 'Foo', 'address_line_2': 'Bar', 'postcode': 'Baz'} + } + letter_request(client, data=data, service_id=sample_trial_letter_template.service_id, + key_type=KEY_TYPE_TEST) + job = Job.query.one() + update_job_task.assert_called_once_with([str(job.id)], queue='research-mode-tasks') + assert not build_dvla_task.called