Merge branch 'master' into inbound-sms

Conflicts:
	app/notifications/receive_notifications.py
	tests/app/notifications/test_receive_notification.py

app/dao/jobs_dao.py

@@ -53,7 +53,8 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
 def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
     query_filter = [
         Job.service_id == service_id,
-        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME']
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
     ]
     if limit_days is not None:
         query_filter.append(cast(Job.created_at, sql_date) >= days_ago(limit_days))

app/dao/notification_usage_dao.py

@@ -153,3 +153,70 @@ def rate_multiplier():
         (NotificationHistory.rate_multiplier == None, literal_column("'1'")),  # noqa
         (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier),  # noqa
     ]), Integer())
+
+
+@statsd(namespace="dao")
+def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id):
+
+    billable_units = 0
+    total_cost = 0.0
+
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    for rate_boundary in rate_boundaries:
+        result = db.session.query(
+            func.sum(
+                NotificationHistory.billable_units * func.coalesce(NotificationHistory.rate_multiplier, 1)
+            ).label('billable_units')
+        ).filter(
+            NotificationHistory.service_id == service_id,
+            NotificationHistory.notification_type == 'sms',
+            NotificationHistory.created_at >= rate_boundary['start_date'],
+            NotificationHistory.created_at < rate_boundary['end_date'],
+            NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE)
+        )
+        billable_units_by_rate_boundry = result.scalar()
+        if billable_units_by_rate_boundry:
+            billable_units += int(billable_units_by_rate_boundry)
+            total_cost += int(billable_units_by_rate_boundry) * rate_boundary['rate']
+
+    return billable_units, total_cost
+
+
+def discover_rate_bounds_for_billing_query(start_date, end_date):
+    bounds = []
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+
+    def current_valid_from(index):
+        return rates[index].valid_from
+
+    def next_valid_from(index):
+        return rates[index + 1].valid_from
+
+    def current_rate(index):
+        return rates[index].rate
+
+    def append_rate(rate_start_date, rate_end_date, rate):
+        bounds.append({
+            'start_date': rate_start_date,
+            'end_date': rate_end_date,
+            'rate': rate
+        })
+
+    if len(rates) == 1:
+        append_rate(start_date, end_date, current_rate(0))
+        return bounds
+
+    for i in range(len(rates)):
+        # first boundary
+        if i == 0:
+            append_rate(start_date, next_valid_from(i), current_rate(i))
+
+        # last boundary
+        elif i == (len(rates) - 1):
+            append_rate(current_valid_from(i), end_date, current_rate(i))
+
+        # other boundaries
+        else:
+            append_rate(current_valid_from(i), next_valid_from(i), current_rate(i))
+
+    return bounds

app/dao/notifications_dao.py

@@ -9,7 +9,8 @@ from flask import current_app
 from notifications_utils.recipients import (
     validate_and_format_phone_number,
     validate_and_format_email_address,
-    InvalidPhoneError
+    InvalidPhoneError,
+    InvalidEmailError,
 )
 from werkzeug.datastructures import MultiDict
 from sqlalchemy import (desc, func, or_, and_, asc)
@@ -477,7 +478,10 @@ def dao_get_notifications_by_to_field(service_id, search_term, statuses=None):
     try:
         normalised = validate_and_format_phone_number(search_term)
     except InvalidPhoneError:
-        normalised = validate_and_format_email_address(search_term)
+        try:
+            normalised = validate_and_format_email_address(search_term)
+        except InvalidEmailError:
+            normalised = search_term
 
     filters = [
         Notification.service_id == service_id,

app/dao/services_dao.py

@@ -3,6 +3,7 @@ from datetime import date, datetime, timedelta
 
 from sqlalchemy import asc, func
 from sqlalchemy.orm import joinedload
+from flask import current_app
 
 from app import db
 from app.dao.dao_utils import (
@@ -137,6 +138,12 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
 @transactional
 @version_class(Service)
 def dao_create_service(service, user, service_id=None, service_permissions=[SMS_TYPE, EMAIL_TYPE]):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.
+    if not service.sms_sender:
+        service.sms_sender = current_app.config['FROM_NUMBER']
+
     from app.dao.permissions_dao import permission_dao
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)