Merge branch 'master' into load-service-on-auth

app/dao/notifications_dao.py

@@ -456,3 +456,10 @@ def dao_update_notifications_sent_to_dvla(job_id, provider):
         {'status': NOTIFICATION_SENDING, "sent_by": provider, "sent_at": now, "updated_at": now})
 
     return updated_count
+
+
+@statsd(namespace="dao")
+def dao_get_notifications_by_to_field(service_id, search_term):
+    return Notification.query.filter(
+        Notification.service_id == service_id,
+        func.replace(func.lower(Notification.to), " ", "") == search_term.lower().replace(" ", "")).all()

app/dao/services_dao.py

@@ -140,7 +140,8 @@ def dao_update_service(service):
     db.session.add(service)
 
 
-def dao_add_user_to_service(service, user, permissions=[]):
+def dao_add_user_to_service(service, user, permissions=None):
+    permissions = permissions or []
     try:
         from app.dao.permissions_dao import permission_dao
         service.users.append(user)
@@ -227,7 +228,8 @@ def fetch_todays_total_message_count(service_id):
 def _stats_for_service_query(service_id):
     return db.session.query(
         Notification.notification_type,
-        Notification.status,
+        # see dao_fetch_todays_stats_for_all_services for why we have this label
+        Notification.status.label('status'),
         func.count(Notification.id).label('count')
     ).filter(
         Notification.service_id == service_id,
@@ -245,13 +247,13 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year)
     start_date, end_date = get_financial_year(year)
     sq = db.session.query(
         NotificationHistory.template_id,
-        NotificationHistory.status,
+        # see dao_fetch_todays_stats_for_all_services for why we have this label
+        NotificationHistory.status.label('status'),
         month.label('month'),
         func.count().label('count')
     ).filter(
         NotificationHistory.service_id == service_id,
         NotificationHistory.created_at.between(start_date, end_date)
-
     ).group_by(
         month,
         NotificationHistory.template_id,
@@ -262,7 +264,7 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year)
         Template.id.label('template_id'),
         Template.name,
         Template.template_type,
-        sq.c.status,
+        sq.c.status.label('status'),
         sq.c.count.label('count'),
         sq.c.month
     ).join(
@@ -280,7 +282,8 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year):
     start_date, end_date = get_financial_year(year)
     rows = db.session.query(
         NotificationHistory.notification_type,
-        NotificationHistory.status,
+        # see dao_fetch_todays_stats_for_all_services for why we have this label
+        NotificationHistory.status.label('status'),
         month,
         func.count(NotificationHistory.id).label('count')
     ).filter(
@@ -319,7 +322,9 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year):
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True):
     query = db.session.query(
         Notification.notification_type,
-        Notification.status,
+        # this label is necessary as the column has a different name under the hood (_status_enum / _status_fkey),
+        # if we query the Notification object there is a hybrid property to translate, but here there isn't anything.
+        Notification.status.label('status'),
         Notification.service_id,
         func.count(Notification.id).label('count')
     ).filter(
@@ -349,7 +354,8 @@ def fetch_stats_by_date_range_for_all_services(start_date, end_date, include_fro
 
     query = db.session.query(
         table.notification_type,
-        table.status,
+        # see dao_fetch_todays_stats_for_all_services for why we have this label
+        table.status.label('status'),
         table.service_id,
         func.count(table.id).label('count')
     ).filter(

app/errors.py

@@ -92,7 +92,7 @@ def register_errors(blueprint):
     @blueprint.errorhandler(SQLAlchemyError)
     def db_error(e):
         current_app.logger.exception(e)
-        if e.orig.pgerror and \
+        if hasattr(e, 'orig') and hasattr(e.orig, 'pgerror') and e.orig.pgerror and \
             ('duplicate key value violates unique constraint "services_name_key"' in e.orig.pgerror or
                 'duplicate key value violates unique constraint "services_email_from_key"' in e.orig.pgerror):
             return jsonify(

app/models.py

@@ -1,8 +1,9 @@
 import time
 import uuid
 import datetime
-from flask import url_for
+from flask import url_for, current_app
 
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.dialects.postgresql import (
     UUID,
     JSON
@@ -46,7 +47,12 @@ class HistoryModel:
 
     def update_from_original(self, original):
         for c in self.__table__.columns:
-            setattr(self, c.name, getattr(original, c.name))
+            # in some cases, columns may have different names to their underlying db column -  so only copy those
+            # that we can, and leave it up to subclasses to deal with any oddities/properties etc.
+            if hasattr(original, c.name):
+                setattr(self, c.name, getattr(original, c.name))
+            else:
+                current_app.logger.debug('{} has no column {} to copy from'.format(original, c.name))
 
 
 class User(db.Model):
@@ -621,6 +627,12 @@ NOTIFICATION_STATUS_TYPES = [
 NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type')
 
 
+class NotificationStatusTypes(db.Model):
+    __tablename__ = 'notification_status_types'
+
+    name = db.Column(db.String(255), primary_key=True)
+
+
 class Notification(db.Model):
     __tablename__ = 'notifications'
 
@@ -656,7 +668,15 @@ class Notification(db.Model):
         unique=False,
         nullable=True,
         onupdate=datetime.datetime.utcnow)
-    status = db.Column(NOTIFICATION_STATUS_TYPES_ENUM, index=True, nullable=False, default='created')
+    _status_enum = db.Column('status', NOTIFICATION_STATUS_TYPES_ENUM, index=True, nullable=False, default='created')
+    _status_fkey = db.Column(
+        'notification_status',
+        db.String,
+        db.ForeignKey('notification_status_types.name'),
+        index=True,
+        nullable=True,
+        default='created'
+    )
     reference = db.Column(db.String, nullable=True, index=True)
     client_reference = db.Column(db.String, index=True, nullable=True)
     _personalisation = db.Column(db.String, nullable=True)
@@ -672,6 +692,15 @@ class Notification(db.Model):
     phone_prefix = db.Column(db.String, nullable=True)
     rate_multiplier = db.Column(db.Float(asdecimal=False), nullable=True)
 
+    @hybrid_property
+    def status(self):
+        return self._status_enum
+
+    @status.setter
+    def status(self, status):
+        self._status_fkey = status
+        self._status_enum = status
+
     @property
     def personalisation(self):
         if self._personalisation:
@@ -844,7 +873,15 @@ class NotificationHistory(db.Model, HistoryModel):
     sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True)
     sent_by = db.Column(db.String, nullable=True)
     updated_at = db.Column(db.DateTime, index=False, unique=False, nullable=True)
-    status = db.Column(NOTIFICATION_STATUS_TYPES_ENUM, index=True, nullable=False, default='created')
+    _status_enum = db.Column('status', NOTIFICATION_STATUS_TYPES_ENUM, index=True, nullable=False, default='created')
+    _status_fkey = db.Column(
+        'notification_status',
+        db.String,
+        db.ForeignKey('notification_status_types.name'),
+        index=True,
+        nullable=True,
+        default='created'
+    )
     reference = db.Column(db.String, nullable=True, index=True)
     client_reference = db.Column(db.String, nullable=True)
 
@@ -855,8 +892,22 @@ class NotificationHistory(db.Model, HistoryModel):
     @classmethod
     def from_original(cls, notification):
         history = super().from_original(notification)
+        history.status = notification.status
         return history
 
+    def update_from_original(self, original):
+        super().update_from_original(original)
+        self.status = original.status
+
+    @hybrid_property
+    def status(self):
+        return self._status_enum
+
+    @status.setter
+    def status(self, status):
+        self._status_fkey = status
+        self._status_enum = status
+
 
 INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled']
 

app/schema_validation/__init__.py

@@ -10,13 +10,13 @@ def validate(json_to_validate, schema):
 
     @format_checker.checks('phone_number', raises=InvalidPhoneError)
     def validate_schema_phone_number(instance):
-        if instance is not None:
+        if isinstance(instance, str):
             validate_phone_number(instance, international=True)
         return True
 
     @format_checker.checks('email_address', raises=InvalidEmailError)
     def validate_schema_email_address(instance):
-        if instance is not None:
+        if isinstance(instance, str):
             validate_email_address(instance)
         return True
 

app/schemas.py

@@ -220,7 +220,9 @@ class NotificationModelSchema(BaseSchema):
     class Meta:
         model = models.Notification
         strict = True
-        exclude = ('_personalisation', 'job', 'service', 'template', 'api_key', '')
+        exclude = ('_personalisation', 'job', 'service', 'template', 'api_key', '_status_enum', '_status_fkey')
+
+    status = fields.String(required=False)
 
 
 class BaseTemplateSchema(BaseSchema):
@@ -315,6 +317,7 @@ class NotificationSchema(ma.Schema):
     class Meta:
         strict = True
 
+    status = fields.String(required=False)
     personalisation = fields.Dict(required=False)
 
 
@@ -369,7 +372,7 @@ class NotificationWithTemplateSchema(BaseSchema):
     class Meta:
         model = models.Notification
         strict = True
-        exclude = ('_personalisation',)
+        exclude = ('_personalisation', '_status_enum', '_status_fkey')
 
     template = fields.Nested(
         TemplateSchema,
@@ -377,6 +380,7 @@ class NotificationWithTemplateSchema(BaseSchema):
         dump_only=True
     )
     job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True)
+    status = fields.String(required=False)
     personalisation = fields.Dict(required=False)
     key_type = field_for(models.Notification, 'key_type', required=True)
     key_name = fields.String()
@@ -492,6 +496,7 @@ class NotificationsFilterSchema(ma.Schema):
     include_from_test_key = fields.Boolean(required=False)
     older_than = fields.UUID(required=False)
     format_for_csv = fields.String()
+    to = fields.String()
 
     @pre_load
     def handle_multidict(self, in_data):

app/service/rest.py

@@ -242,6 +242,8 @@ def get_service_history(service_id):
 @service_blueprint.route('/<uuid:service_id>/notifications', methods=['GET'])
 def get_all_notifications_for_service(service_id):
     data = notifications_filter_schema.load(request.args).data
+    if data.get("to", None):
+        return search_for_notification_by_to_field(service_id, request.query_string.decode())
     page = data['page'] if 'page' in data else 1
     page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
     limit_days = data.get('limit_days')
@@ -271,6 +273,13 @@ def get_all_notifications_for_service(service_id):
     ), 200
 
 
+def search_for_notification_by_to_field(service_id, search_term):
+    search_term = search_term.replace('to=', '')
+    results = notifications_dao.dao_get_notifications_by_to_field(service_id, search_term)
+    return jsonify(
+        notifications=notification_with_template_schema.dump(results, many=True).data), 200
+
+
 @service_blueprint.route('/<uuid:service_id>/notifications/monthly', methods=['GET'])
 def get_monthly_notification_stats(service_id):
     service = dao_fetch_service_by_id(service_id)