Merge branch 'master' into celery_logging

app/celery/scheduled_tasks.py

@@ -8,7 +8,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app.aws import s3
 from app import notify_celery
-from app.performance_platform import total_sent_notifications
+from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
@@ -176,27 +176,32 @@ def timeout_notifications():
 @statsd(namespace="tasks")
 def send_daily_performance_platform_stats():
     if performance_platform_client.active:
-        count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
-        email_sent_count = count_dict.get('email').get('count')
-        sms_sent_count = count_dict.get('sms').get('count')
-        start_date = count_dict.get('start_date')
+        send_total_sent_notifications_to_performance_platform()
+        processing_time.send_processing_time_to_performance_platform()
 
-        current_app.logger.info(
-            "Attempting to update performance platform for date {} with email count {} and sms count {}"
-            .format(start_date, email_sent_count, sms_sent_count)
-        )
 
-        total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            start_date,
-            'sms',
-            sms_sent_count
-        )
+def send_total_sent_notifications_to_performance_platform():
+    count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
+    email_sent_count = count_dict.get('email').get('count')
+    sms_sent_count = count_dict.get('sms').get('count')
+    start_date = count_dict.get('start_date')
 
-        total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            start_date,
-            'email',
-            email_sent_count
-        )
+    current_app.logger.info(
+        "Attempting to update performance platform for date {} with email count {} and sms count {}"
+        .format(start_date, email_sent_count, sms_sent_count)
+    )
+
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_date,
+        'sms',
+        sms_sent_count
+    )
+
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_date,
+        'email',
+        email_sent_count
+    )
 
 
 @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')

app/celery/tasks.py

@@ -86,9 +86,11 @@ def process_job(job_id):
         process_row(row_number, recipient, personalisation, template, job, service)
 
     if template.template_type == LETTER_TYPE:
-        build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
-        # temporary logging
-        current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id))
+        if service.research_mode:
+            update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
+        else:
+            build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
+            current_app.logger.info("send job {} to build-dvla-file in the {} queue".format(job_id, QueueNames.JOBS))
     else:
         job.job_status = JOB_STATUS_FINISHED
 

app/commands.py

@@ -1,7 +1,7 @@
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from decimal import Decimal
-from flask_script import Command, Manager, Option
+from flask_script import Command, Option
 
 from app import db
 from app.dao.monthly_billing_dao import (
@@ -16,6 +16,8 @@ from app.dao.services_dao import (
 )
 from app.dao.provider_rates_dao import create_provider_rates
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
+from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
 
 
 class CreateProviderRateCommand(Command):
@@ -49,7 +51,7 @@ class PurgeFunctionalTestDataCommand(Command):
         Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix."),
     )
 
-    def run(self, service_name_prefix=None, user_email_prefix=None):
+    def run(self, user_email_prefix=None):
         if user_email_prefix:
             users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
             for usr in users:
@@ -167,33 +169,62 @@ class CustomDbScript(Command):
 
 
 class PopulateMonthlyBilling(Command):
-        option_list = (
-            Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"),
+    option_list = (
+        Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"),
+    )
+
+    def run(self, year):
+        service_ids = get_service_ids_that_need_billing_populated(
+            start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
         )
+        start, end = 1, 13
+        if year == '2016':
+            start = 4
 
-        def run(self, year):
-            service_ids = get_service_ids_that_need_billing_populated(
-                start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
-            )
-            start, end = 1, 13
-            if year == '2016':
-                start = 4
+        for service_id in service_ids:
+            print('Starting to populate data for service {}'.format(str(service_id)))
+            print('Starting populating monthly billing for {}'.format(year))
+            for i in range(start, end):
+                print('Population for {}-{}'.format(i, year))
+                self.populate(service_id, year, i)
 
-            for service_id in service_ids:
-                print('Starting to populate data for service {}'.format(str(service_id)))
-                print('Starting populating monthly billing for {}'.format(year))
-                for i in range(start, end):
-                    print('Population for {}-{}'.format(i, year))
-                    self.populate(service_id, year, i)
+    def populate(self, service_id, year, month):
+        create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1))
+        sms_res = get_monthly_billing_by_notification_type(
+            service_id, datetime(int(year), int(month), 1), SMS_TYPE
+        )
+        email_res = get_monthly_billing_by_notification_type(
+            service_id, datetime(int(year), int(month), 1), EMAIL_TYPE
+        )
+        print("Finished populating data for {} for service id {}".format(month, str(service_id)))
+        print('SMS: {}'.format(sms_res.monthly_totals))
+        print('Email: {}'.format(email_res.monthly_totals))
 
-        def populate(self, service_id, year, month):
-            create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1))
-            sms_res = get_monthly_billing_by_notification_type(
-                service_id, datetime(int(year), int(month), 1), SMS_TYPE
-            )
-            email_res = get_monthly_billing_by_notification_type(
-                service_id, datetime(int(year), int(month), 1), EMAIL_TYPE
-            )
-            print("Finished populating data for {} for service id {}".format(month, str(service_id)))
-            print('SMS: {}'.format(sms_res.monthly_totals))
-            print('Email: {}'.format(email_res.monthly_totals))
+
+class BackfillProcessingTime(Command):
+    option_list = (
+        Option('-s', '--start_date', dest='start_date', help="Date (%Y-%m-%d) start date inclusive"),
+        Option('-e', '--end_date', dest='end_date', help="Date (%Y-%m-%d) end date inclusive"),
+    )
+
+    def run(self, start_date, end_date):
+        start_date = datetime.strptime(start_date, '%Y-%m-%d')
+        end_date = datetime.strptime(end_date, '%Y-%m-%d')
+
+        delta = end_date - start_date
+
+        print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+        for i in range(delta.days + 1):
+            # because the tz conversion funcs talk about midnight, and the midnight before last,
+            # we want to pretend we're running this from the next morning, so add one.
+            process_date = start_date + timedelta(days=i + 1)
+
+            process_start_date = get_midnight_for_day_before(process_date)
+            process_end_date = get_london_midnight_in_utc(process_date)
+
+            print('Sending notification processing-time for {} - {}'.format(
+                process_start_date.isoformat(),
+                process_end_date.isoformat()
+            ))
+            send_processing_time_for_start_and_end(process_start_date, process_end_date)

app/dao/monthly_billing_dao.py

@@ -1,7 +1,5 @@
 from datetime import datetime
 
-from sqlalchemy import func
-
 
 from app import db
 from app.dao.dao_utils import transactional

app/dao/notification_usage_dao.py

@@ -1,4 +1,3 @@
-from collections import namedtuple
 from datetime import datetime, timedelta
 
 from sqlalchemy import Float, Integer
@@ -17,41 +16,7 @@ from app.models import (
     EMAIL_TYPE
 )
 from app.statsd_decorators import statsd
-from app.utils import get_london_month_from_utc_column, convert_utc_to_bst
-
-
-@statsd(namespace="dao")
-def get_yearly_billing_data(service_id, year):
-    start_date, end_date = get_financial_year(year)
-    rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
-
-    if not rates:
-        return []
-
-    def get_valid_from(valid_from):
-        return start_date if valid_from < start_date else valid_from
-
-    result = []
-    for r, n in zip(rates, rates[1:]):
-        result.append(
-            sms_yearly_billing_data_query(
-                r.rate,
-                service_id,
-                get_valid_from(r.valid_from),
-                n.valid_from
-            )
-        )
-    result.append(
-        sms_yearly_billing_data_query(
-            rates[-1].rate,
-            service_id,
-            get_valid_from(rates[-1].valid_from),
-            end_date
-        )
-    )
-
-    result.append(email_yearly_billing_data_query(service_id, start_date, end_date))
-    return sum(result, [])
+from app.utils import get_london_month_from_utc_column
 
 
 @statsd(namespace="dao")
@@ -112,52 +77,6 @@ def billing_data_filter(notification_type, start_date, end_date, service_id):
     ]
 
 
-def email_yearly_billing_data_query(service_id, start_date, end_date, rate=0):
-    result = db.session.query(
-        func.count(NotificationHistory.id),
-        func.count(NotificationHistory.id),
-        rate_multiplier(),
-        NotificationHistory.notification_type,
-        NotificationHistory.international,
-        cast(rate, Integer())
-    ).filter(
-        *billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id)
-    ).group_by(
-        NotificationHistory.notification_type,
-        rate_multiplier(),
-        NotificationHistory.international
-    ).first()
-
-    if not result:
-        return [(0, 0, 1, EMAIL_TYPE, False, 0)]
-    else:
-        return [result]
-
-
-def sms_yearly_billing_data_query(rate, service_id, start_date, end_date):
-    result = db.session.query(
-        cast(func.sum(NotificationHistory.billable_units * rate_multiplier()), Integer()),
-        func.sum(NotificationHistory.billable_units),
-        rate_multiplier(),
-        NotificationHistory.notification_type,
-        NotificationHistory.international,
-        cast(rate, Float())
-    ).filter(
-        *billing_data_filter(SMS_TYPE, start_date, end_date, service_id)
-    ).group_by(
-        NotificationHistory.notification_type,
-        NotificationHistory.international,
-        rate_multiplier()
-    ).order_by(
-        rate_multiplier()
-    ).all()
-
-    if not result:
-        return [(0, 0, 1, SMS_TYPE, False, rate)]
-    else:
-        return result
-
-
 def get_rates_for_daterange(start_date, end_date, notification_type):
     rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all()
 

app/dao/notifications_dao.py

@@ -16,6 +16,8 @@ from notifications_utils.recipients import (
 from werkzeug.datastructures import MultiDict
 from sqlalchemy import (desc, func, or_, and_, asc)
 from sqlalchemy.orm import joinedload
+from sqlalchemy.sql.expression import case
+from sqlalchemy.sql import functions
 from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 
 from app import db, create_uuid
@@ -42,7 +44,6 @@ from app.models import (
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
-from app.utils import get_london_month_from_utc_column
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -519,3 +520,38 @@ def set_scheduled_notification_to_processed(notification_id):
         {'pending': False}
     )
     db.session.commit()
+
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+    """
+    SELECT
+    count(notification_history),
+    coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
+    FROM notification_history
+    WHERE
+    created_at > 'START DATE' AND
+    created_at < 'END DATE' AND
+    api_key_id IS NOT NULL AND
+    key_type != 'test' AND
+    notification_type != 'letter';
+    """
+    under_10_secs = NotificationHistory.sent_at - NotificationHistory.created_at <= timedelta(seconds=10)
+    sum_column = functions.coalesce(functions.sum(
+        case(
+            [
+                (under_10_secs, 1)
+            ],
+            else_=0
+        )
+    ), 0)
+
+    return db.session.query(
+        func.count(NotificationHistory.id).label('messages_total'),
+        sum_column.label('messages_within_10_secs')
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at < end_date,
+        NotificationHistory.api_key_id.isnot(None),
+        NotificationHistory.key_type != KEY_TYPE_TEST,
+        NotificationHistory.notification_type != LETTER_TYPE
+    ).one()

app/dao/service_email_reply_to_dao.py

@@ -0,0 +1,32 @@
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import ServiceEmailReplyTo
+
+
+def create_or_update_email_reply_to(service_id, email_address):
+    reply_to = dao_get_reply_to_by_service_id(service_id)
+    if reply_to:
+        reply_to.email_address = email_address
+        dao_update_reply_to_email(reply_to)
+    else:
+        reply_to = ServiceEmailReplyTo(service_id=service_id, email_address=email_address)
+        dao_create_reply_to_email_address(reply_to)
+
+
+@transactional
+def dao_create_reply_to_email_address(reply_to_email):
+    db.session.add(reply_to_email)
+
+
+def dao_get_reply_to_by_service_id(service_id):
+    reply_to = db.session.query(
+        ServiceEmailReplyTo
+    ).filter(
+        ServiceEmailReplyTo.service_id == service_id
+    ).first()
+    return reply_to
+
+
+@transactional
+def dao_update_reply_to_email(reply_to):
+    db.session.add(reply_to)

app/job/rest.py

@@ -32,7 +32,7 @@ from app.schemas import (
 
 from app.celery.tasks import process_job
 
-from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED
+from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED, LETTER_TYPE
 
 from app.utils import pagination_links
 
@@ -190,6 +190,9 @@ def create_job(service_id):
     })
     template = dao_get_template_by_id(data['template'])
 
+    if template.template_type == LETTER_TYPE and service.restricted:
+        raise InvalidRequest("Create letter job is not allowed for service in trial mode ", 403)
+
     errors = unarchived_template_schema.validate({'archived': template.archived})
 
     if errors:

app/models.py

@@ -245,7 +245,7 @@ class Service(db.Model, Versioned):
         if self.inbound_number and self.inbound_number.active:
             return self.inbound_number.number
         else:
-            return self.sms_sender or current_app.config['FROM_NUMBER']
+            return self.sms_sender
 
 
 class InboundNumber(db.Model):
@@ -278,6 +278,21 @@ class InboundNumber(db.Model):
         }
 
 
+class ServiceSmsSender(db.Model):
+    __tablename__ = "service_sms_senders"
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    sms_sender = db.Column(db.String(11), nullable=False)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=True, index=True, nullable=False)
+    service = db.relationship(Service, backref=db.backref("service_sms_senders", uselist=False))
+    is_default = db.Column(db.Boolean, nullable=False, default=True)
+    inbound_number_id = db.Column(UUID(as_uuid=True), db.ForeignKey('inbound_numbers.id'),
+                                  unique=True, index=True, nullable=True)
+    inbound_number = db.relationship(InboundNumber, backref=db.backref("inbound_number", uselist=False))
+    created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
+
+
 class ServicePermission(db.Model):
     __tablename__ = "service_permissions"
 
@@ -318,7 +333,7 @@ class ServiceWhitelist(db.Model):
 
         try:
             if recipient_type == MOBILE_TYPE:
-                validate_phone_number(recipient)
+                validate_phone_number(recipient, international=True)
                 instance.recipient = recipient
             elif recipient_type == EMAIL_TYPE:
                 validate_email_address(recipient)
@@ -1314,3 +1329,17 @@ class MonthlyBilling(db.Model):
 
     def __repr__(self):
         return str(self.serialized())
+
+
+class ServiceEmailReplyTo(db.Model):
+    __tablename__ = "service_email_reply_to"
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
+    service = db.relationship(Service, backref=db.backref("reply_to_email_addresses", uselist=False))
+
+    email_address = db.Column(db.Text, nullable=False, index=False, unique=False)
+    is_default = db.Column(db.Boolean, nullable=False, default=True)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)

app/performance_platform/processing_time.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from flask import current_app
+
+from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
+from app.dao.notifications_dao import dao_get_total_notifications_sent_per_day_for_performance_platform
+from app import performance_platform_client
+
+
+def send_processing_time_to_performance_platform():
+    today = datetime.utcnow()
+    start_date = get_midnight_for_day_before(today)
+    end_date = get_london_midnight_in_utc(today)
+
+    send_processing_time_for_start_and_end(start_date, end_date)
+
+
+def send_processing_time_for_start_and_end(start_date, end_date):
+    result = dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date)
+
+    current_app.logger.info(
+        'Sending processing-time to performance platform for date {}. Total: {}, under 10 secs {}'.format(
+            start_date, result.messages_total, result.messages_within_10_secs
+        )
+    )
+
+    send_processing_time_data(start_date, 'messages-total', result.messages_total)
+    send_processing_time_data(start_date, 'messages-within-10-secs', result.messages_within_10_secs)
+
+
+def send_processing_time_data(date, status, count):
+    payload = performance_platform_client.format_payload(
+        dataset='processing-time',
+        date=date,
+        group_name='status',
+        group_value=status,
+        count=count
+    )
+
+    performance_platform_client.send_stats_to_performance_platform(payload)

app/schemas.py

@@ -130,7 +130,7 @@ class UserUpdateAttributeSchema(BaseSchema):
     @validates('mobile_number')
     def validate_mobile_number(self, value):
         try:
-            validate_phone_number(value)
+            validate_phone_number(value, international=True)
         except InvalidPhoneError as error:
             raise ValidationError('Invalid phone number: {}'.format(error))
 

app/service/rest.py

@@ -1,5 +1,4 @@
 import itertools
-import json
 from datetime import datetime
 
 from flask import (
@@ -11,7 +10,7 @@ from flask import (
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm.exc import NoResultFound
 
-from app.dao import notification_usage_dao, notifications_dao
+from app.dao import notifications_dao
 from app.dao.dao_utils import dao_rollback
 from app.dao.api_key_dao import (
     save_model_api_key,
@@ -46,6 +45,7 @@ from app.dao.service_whitelist_dao import (
     dao_add_and_commit_whitelisted_contacts,
     dao_remove_service_whitelist
 )
+from app.dao.service_email_reply_to_dao import create_or_update_email_reply_to
 from app.dao.provider_statistics_dao import get_fragment_count
 from app.dao.users_dao import get_user_by_id
 from app.errors import (
@@ -132,9 +132,13 @@ def create_service():
 
 @service_blueprint.route('/<uuid:service_id>', methods=['POST'])
 def update_service(service_id):
+    req_json = request.get_json()
     fetched_service = dao_fetch_service_by_id(service_id)
     # Capture the status change here as Marshmallow changes this later
-    service_going_live = fetched_service.restricted and not request.get_json().get('restricted', True)
+    service_going_live = fetched_service.restricted and not req_json.get('restricted', True)
+
+    if 'reply_to_email_address' in req_json:
+        create_or_update_email_reply_to(fetched_service.id, req_json['reply_to_email_address'])
 
     current_data = dict(service_schema.dump(fetched_service).data.items())
     current_data.update(request.get_json())
@@ -458,43 +462,6 @@ def get_monthly_template_stats(service_id):
         raise InvalidRequest('Year must be a number', status_code=400)
 
 
-@service_blueprint.route('/<uuid:service_id>/yearly-usage')
-def get_yearly_billing_usage(service_id):
-    try:
-        year = int(request.args.get('year'))
-        results = notification_usage_dao.get_yearly_billing_data(service_id, year)
-        json_result = [{
-            "credits": x[0],
-            "billing_units": x[1],
-            "rate_multiplier": x[2],
-            "notification_type": x[3],
-            "international": x[4],
-            "rate": x[5]
-        } for x in results]
-        return json.dumps(json_result)
-
-    except TypeError:
-        return jsonify(result='error', message='No valid year provided'), 400
-
-
-@service_blueprint.route('/<uuid:service_id>/monthly-usage')
-def get_yearly_monthly_usage(service_id):
-    try:
-        year = int(request.args.get('year'))
-        results = notification_usage_dao.get_monthly_billing_data(service_id, year)
-        json_results = [{
-            "month": x[0],
-            "billing_units": x[1],
-            "rate_multiplier": x[2],
-            "international": x[3],
-            "notification_type": x[4],
-            "rate": x[5]
-        } for x in results]
-        return json.dumps(json_results)
-    except TypeError:
-        return jsonify(result='error', message='No valid year provided'), 400
-
-
 @service_blueprint.route('/<uuid:service_id>/inbound-api', methods=['POST'])
 def create_service_inbound_api(service_id):
     data = request.get_json()

app/utils.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 import pytz
 from flask import url_for
 from sqlalchemy import func
-from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate, LetterPreviewTemplate
+from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate
 
 local_timezone = pytz.timezone("Europe/London")
 

app/v2/notifications/post_notifications.py

@@ -150,15 +150,16 @@ def process_letter_notification(*, letter_data, api_key, template):
     if api_key.key_type == KEY_TYPE_TEAM:
         raise BadRequestError(message='Cannot send letters with a team api key', status_code=403)
 
+    if api_key.service.restricted and api_key.key_type != KEY_TYPE_TEST:
+            raise BadRequestError(message='Cannot send letters when service is in trial mode', status_code=403)
+
     job = create_letter_api_job(template)
     notification = create_letter_notification(letter_data, job, api_key)
 
     if api_key.service.research_mode or api_key.key_type == KEY_TYPE_TEST:
-
         # distinguish real API jobs from test jobs by giving the test jobs a different filename
         job.original_file_name = LETTER_TEST_API_FILENAME
         dao_update_job(job)
-
         update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
     else:
         build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)