Merge pull request #217 from alphagov/update_uuid_all_db_models

Update uuid all db models

app/celery/tasks.py

@@ -70,7 +70,7 @@ def delete_verify_codes():
 def delete_successful_notifications():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago('sent')
+        deleted = delete_notifications_created_more_than_a_week_ago('sending')
         current_app.logger.info(
             "Delete job started {} finished {} deleted {} successful notifications".format(
                 start,
@@ -129,12 +129,13 @@ def process_job(job_id):
     if stats:
         total_sent = stats.emails_requested + stats.sms_requested
 
-    if total_sent + job.notification_count > service.limit:
+    if total_sent + job.notification_count > service.message_limit:
         job.status = 'sending limits exceeded'
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
         current_app.logger.info(
-            "Job {} size {} error. Sending limits {} exceeded".format(job_id, job.notification_count, service.limit)
+            "Job {} size {} error. Sending limits {} exceeded".format(
+                job_id, job.notification_count, service.message_limit)
         )
         return
 
@@ -152,7 +153,7 @@ def process_job(job_id):
     ).recipients_and_personalisation:
 
         encrypted = encryption.encrypt({
-            'template': template.id,
+            'template': str(template.id),
             'job': str(job.id),
             'to': recipient,
             'personalisation': personalisation
@@ -217,7 +218,7 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
             to=notification['to'],
             service_id=service_id,
             job_id=notification.get('job', None),
-            status='failed' if restricted else 'sent',
+            status='failed' if restricted else 'sending',
             created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
             sent_by=client.get_name()
@@ -277,7 +278,7 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
             to=notification['to'],
             service_id=service_id,
             job_id=notification.get('job', None),
-            status='failed' if restricted else 'sent',
+            status='failed' if restricted else 'sending',
             created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
             sent_by=client.get_name()

app/clients/email/aws_ses.py

@@ -8,7 +8,7 @@ ses_response_map = {
     'Bounce': {
         "message": 'Bounced',
         "success": False,
-        "notification_status": 'bounce',
+        "notification_status": 'failed',
         "notification_statistics_status": STATISTICS_FAILURE
     },
     'Delivery': {
@@ -19,9 +19,9 @@ ses_response_map = {
     },
     'Complaint': {
         "message": 'Complaint',
-        "success": False,
-        "notification_status": 'complaint',
-        "notification_statistics_status": STATISTICS_FAILURE
+        "success": True,
+        "notification_status": 'delivered',
+        "notification_statistics_status": STATISTICS_DELIVERED
     }
 }
 

app/clients/sms/firetext.py

@@ -25,9 +25,9 @@ firetext_responses = {
     },
     '2': {
         "message": 'Undelivered (Pending with Network)',
-        "success": False,
-        "notification_statistics_status": None,
-        "notification_status": 'sent'
+        "success": True,
+        "notification_statistics_status": STATISTICS_DELIVERED,
+        "notification_status": 'delivered'
     }
 }
 

app/clients/sms/twilio.py

@@ -44,8 +44,10 @@ class TwilioClient(SmsClient):
     def status(self, message_id):
         try:
             response = self.client.messages.get(message_id)
-            if response.status in ('delivered', 'undelivered', 'failed'):
+            if response.status in ('delivered', 'failed'):
                 return response.status
+            elif response.status == 'undelivered':
+                return 'sending'
             return None
         except TwilioRestException as e:
             current_app.logger.exception(e)

app/dao/notifications_dao.py

@@ -112,12 +112,12 @@ def update_query(notification_type, status):
         TEMPLATE_TYPE_SMS: {
             STATISTICS_REQUESTED: NotificationStatistics.sms_requested,
             STATISTICS_DELIVERED: NotificationStatistics.sms_delivered,
-            STATISTICS_FAILURE: NotificationStatistics.sms_error
+            STATISTICS_FAILURE: NotificationStatistics.sms_failed
         },
         TEMPLATE_TYPE_EMAIL: {
             STATISTICS_REQUESTED: NotificationStatistics.emails_requested,
             STATISTICS_DELIVERED: NotificationStatistics.emails_delivered,
-            STATISTICS_FAILURE: NotificationStatistics.emails_error
+            STATISTICS_FAILURE: NotificationStatistics.emails_failed
         }
     }
     return {

app/models.py

@@ -1,7 +1,7 @@
 import uuid
 import datetime
 
-from sqlalchemy import UniqueConstraint, Sequence
+from sqlalchemy import UniqueConstraint, Sequence, CheckConstraint
 
 from . import db
 from sqlalchemy.dialects.postgresql import (
@@ -24,7 +24,7 @@ def filter_null_value_fields(obj):
 class User(db.Model):
     __tablename__ = 'users'
 
-    id = db.Column(db.Integer, primary_key=True)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     name = db.Column(db.String, nullable=False, index=True, unique=False)
     email_address = db.Column(db.String(255), nullable=False, index=True, unique=True)
     created_at = db.Column(
@@ -62,7 +62,7 @@ class User(db.Model):
 user_to_service = db.Table(
     'user_to_service',
     db.Model.metadata,
-    db.Column('user_id', db.Integer, db.ForeignKey('users.id')),
+    db.Column('user_id', UUID(as_uuid=True), db.ForeignKey('users.id')),
     db.Column('service_id', UUID(as_uuid=True), db.ForeignKey('services.id')),
 
     UniqueConstraint('user_id', 'service_id', name='uix_user_to_service')
@@ -74,7 +74,6 @@ class Service(db.Model):
     __tablename__ = 'services'
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    old_id = db.Column(db.Integer, Sequence('services_id_seq'), nullable=False)
     name = db.Column(db.String(255), nullable=False, unique=True)
     created_at = db.Column(
         db.DateTime,
@@ -89,7 +88,7 @@ class Service(db.Model):
         nullable=True,
         onupdate=datetime.datetime.now)
     active = db.Column(db.Boolean, index=False, unique=False, nullable=False)
-    limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    message_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
     users = db.relationship(
         'User',
         secondary=user_to_service,
@@ -99,9 +98,9 @@ class Service(db.Model):
 
 
 class ApiKey(db.Model):
-    __tablename__ = 'api_key'
+    __tablename__ = 'api_keys'
 
-    id = db.Column(db.Integer, primary_key=True)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     name = db.Column(db.String(255), nullable=False)
     secret = db.Column(db.String(255), unique=True, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
@@ -116,16 +115,16 @@ class ApiKey(db.Model):
 class NotificationStatistics(db.Model):
     __tablename__ = 'notification_statistics'
 
-    id = db.Column(db.Integer, primary_key=True)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     day = db.Column(db.String(255), nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
     service = db.relationship('Service', backref=db.backref('service_notification_stats', lazy='dynamic'))
     emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
     emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
+    emails_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
     sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
     sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
+    sms_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
 
     __table_args__ = (
         UniqueConstraint('service_id', 'day', name='uix_service_to_day'),
@@ -142,7 +141,7 @@ TEMPLATE_TYPES = [TEMPLATE_TYPE_SMS, TEMPLATE_TYPE_EMAIL, TEMPLATE_TYPE_LETTER]
 class Template(db.Model):
     __tablename__ = 'templates'
 
-    id = db.Column(db.Integer, primary_key=True)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     name = db.Column(db.String(255), nullable=False)
     template_type = db.Column(db.Enum(*TEMPLATE_TYPES, name='template_type'), nullable=False)
     created_at = db.Column(
@@ -173,7 +172,7 @@ class Job(db.Model):
     original_file_name = db.Column(db.String, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
     service = db.relationship('Service', backref=db.backref('jobs', lazy='dynamic'))
-    template_id = db.Column(db.BigInteger, db.ForeignKey('templates.id'), index=True, unique=False)
+    template_id = db.Column(UUID(as_uuid=True), db.ForeignKey('templates.id'), index=True, unique=False)
     template = db.relationship('Template', backref=db.backref('jobs', lazy='dynamic'))
     created_at = db.Column(
         db.DateTime,
@@ -208,8 +207,8 @@ VERIFY_CODE_TYPES = ['email', 'sms']
 class VerifyCode(db.Model):
     __tablename__ = 'verify_codes'
 
-    id = db.Column(db.Integer, primary_key=True)
-    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), index=True, nullable=False)
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    user_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     user = db.relationship('User', backref=db.backref('verify_codes', lazy='dynamic'))
     _code = db.Column(db.String, nullable=False)
     code_type = db.Column(db.Enum(*VERIFY_CODE_TYPES, name='verify_code_types'),
@@ -235,7 +234,7 @@ class VerifyCode(db.Model):
         return check_hash(cde, self._code)
 
 
-NOTIFICATION_STATUS_TYPES = ['sent', 'delivered', 'failed', 'complaint', 'bounce']
+NOTIFICATION_STATUS_TYPES = ['sending', 'delivered', 'failed']
 
 
 class Notification(db.Model):
@@ -248,7 +247,7 @@ class Notification(db.Model):
     job = db.relationship('Job', backref=db.backref('notifications', lazy='dynamic'))
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False)
     service = db.relationship('Service')
-    template_id = db.Column(db.BigInteger, db.ForeignKey('templates.id'), index=True, unique=False)
+    template_id = db.Column(UUID(as_uuid=True), db.ForeignKey('templates.id'), index=True, unique=False)
     template = db.relationship('Template')
     created_at = db.Column(
         db.DateTime,
@@ -268,7 +267,7 @@ class Notification(db.Model):
         nullable=True,
         onupdate=datetime.datetime.utcnow)
     status = db.Column(
-        db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sent')
+        db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sending')
     reference = db.Column(db.String, nullable=True, index=True)
 
 
@@ -281,7 +280,7 @@ class InvitedUser(db.Model):
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     email_address = db.Column(db.String(255), nullable=False)
-    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), index=True, nullable=False)
+    user_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     from_user = db.relationship('User')
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False)
     service = db.relationship('Service')
@@ -332,7 +331,7 @@ class Permission(db.Model):
     # Service id is optional, if the service is omitted we will assume the permission is not service specific.
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=True)
     service = db.relationship('Service')
-    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), index=True, nullable=False)
+    user_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     user = db.relationship('User')
     permission = db.Column(
         db.Enum(*PERMISSION_LIST, name='permission_types'),
@@ -356,7 +355,7 @@ class TemplateStatistics(db.Model):
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
     service = db.relationship('Service', backref=db.backref('template_statistics', lazy='dynamic'))
-    template_id = db.Column(db.BigInteger, db.ForeignKey('templates.id'), index=True, nullable=False, unique=False)
+    template_id = db.Column(UUID(as_uuid=True), db.ForeignKey('templates.id'), index=True, nullable=False, unique=False)
     template = db.relationship('Template')
     usage_count = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=1)
     day = db.Column(db.Date, index=True, nullable=False, unique=False, default=datetime.date.today)

app/notifications/rest.py

@@ -195,6 +195,7 @@ def get_all_notifications():
         api_user['client'],
         filter_dict=data,
         page=page)
+
     return jsonify(
         notifications=notification_status_schema.dump(pagination.items, many=True).data,
         links=pagination_links(
@@ -286,8 +287,9 @@ def send_notification(notification_type):
         total_sms_count = service_stats.sms_requested
         total_email_count = service_stats.emails_requested
 
-        if (total_email_count + total_sms_count >= service.limit) and service.restricted:
-            return jsonify(result="error", message='Exceeded send limits ({}) for today'.format(service.limit)), 429
+        if (total_email_count + total_sms_count >= service.message_limit) and service.restricted:
+            return jsonify(result="error", message='Exceeded send limits ({}) for today'.format(
+                service.message_limit)), 429
 
     notification, errors = (
         sms_template_notification_schema if notification_type == 'sms' else email_notification_schema

app/schemas.py

@@ -91,12 +91,23 @@ class NotificationModelSchema(BaseSchema):
         model = models.Notification
 
 
-class TemplateSchema(BaseSchema):
+class BaseTemplateSchema(BaseSchema):
     class Meta:
         model = models.Template
         exclude = ("updated_at", "created_at", "service_id", "jobs")
 
 
+class TemplateSchema(BaseTemplateSchema):
+
+    @validates_schema
+    def validate_type(self, data):
+        template_type = data.get('template_type')
+        if template_type and template_type == 'email':
+            subject = data.get('subject')
+            if not subject or subject.strip() == '':
+                raise ValidationError('Invalid template subject', 'subject')
+
+
 class NotificationsStatisticsSchema(BaseSchema):
     class Meta:
         model = models.NotificationStatistics
@@ -140,7 +151,7 @@ class SmsNotificationSchema(NotificationSchema):
 
 class EmailNotificationSchema(NotificationSchema):
     to = fields.Str(required=True)
-    template = fields.Int(required=True)
+    template = fields.Str(required=True)
 
     @validates('to')
     def validate_to(self, value):
@@ -151,17 +162,17 @@ class EmailNotificationSchema(NotificationSchema):
 
 
 class SmsTemplateNotificationSchema(SmsNotificationSchema):
-    template = fields.Int(required=True)
+    template = fields.Str(required=True)
     job = fields.String()
 
 
 class JobSmsTemplateNotificationSchema(SmsNotificationSchema):
-    template = fields.Int(required=True)
+    template = fields.Str(required=True)
     job = fields.String(required=True)
 
 
 class JobEmailTemplateNotificationSchema(EmailNotificationSchema):
-    template = fields.Int(required=True)
+    template = fields.Str(required=True)
     job = fields.String(required=True)
 
 
@@ -220,8 +231,8 @@ class EmailDataSchema(ma.Schema):
 
 
 class NotificationsFilterSchema(ma.Schema):
-    template_type = fields.Nested(TemplateSchema, only='template_type', many=True)
-    status = fields.Nested(NotificationModelSchema, only='status', many=True)
+    template_type = fields.Nested(BaseTemplateSchema, only=['template_type'], many=True)
+    status = fields.Nested(NotificationModelSchema, only=['status'], many=True)
     page = fields.Int(required=False)
 
     @pre_load

app/service/rest.py

@@ -108,7 +108,7 @@ def renew_api_key(service_id=None):
     return jsonify(data=unsigned_api_key), 201
 
 
-@service.route('/<uuid:service_id>/api-key/revoke/<int:api_key_id>', methods=['POST'])
+@service.route('/<uuid:service_id>/api-key/revoke/<uuid:api_key_id>', methods=['POST'])
 def revoke_api_key(service_id, api_key_id):
     service_api_key = get_model_api_keys(service_id=service_id, id=api_key_id)
 
@@ -117,7 +117,7 @@ def revoke_api_key(service_id, api_key_id):
 
 
 @service.route('/<uuid:service_id>/api-keys', methods=['GET'])
-@service.route('/<uuid:service_id>/api-keys/<int:key_id>', methods=['GET'])
+@service.route('/<uuid:service_id>/api-keys/<uuid:key_id>', methods=['GET'])
 def get_api_keys(service_id, key_id=None):
     dao_fetch_service_by_id(service_id=service_id)
 

app/template/rest.py

@@ -47,7 +47,7 @@ def create_template(service_id):
     return jsonify(data=template_schema.dump(new_template).data), 201
 
 
-@template.route('/<int:template_id>', methods=['POST'])
+@template.route('/<uuid:template_id>', methods=['POST'])
 def update_template(service_id, template_id):
     fetched_template = dao_get_template_by_id_and_service_id(template_id=template_id, service_id=service_id)
 
@@ -70,7 +70,7 @@ def get_all_templates_for_service(service_id):
     return jsonify(data=data)
 
 
-@template.route('/<int:template_id>', methods=['GET'])
+@template.route('/<uuid:template_id>', methods=['GET'])
 def get_template_by_id_and_service_id(service_id, template_id):
     fetched_template = dao_get_template_by_id_and_service_id(template_id=template_id, service_id=service_id)
     data, errors = template_schema.dump(fetched_template)

app/user/rest.py

@@ -51,7 +51,7 @@ def create_user():
     return jsonify(data=user_schema.dump(user_to_create).data), 201
 
 
-@user.route('/<int:user_id>', methods=['PUT'])
+@user.route('/<uuid:user_id>', methods=['PUT'])
 def update_user(user_id):
     user_to_update = get_model_users(user_id=user_id)
     req_json = request.get_json()
@@ -68,7 +68,7 @@ def update_user(user_id):
     return jsonify(data=user_schema.dump(user_to_update).data), status_code
 
 
-@user.route('/<int:user_id>/verify/password', methods=['POST'])
+@user.route('/<uuid:user_id>/verify/password', methods=['POST'])
 def verify_user_password(user_id):
     user_to_verify = get_model_users(user_id=user_id)
 
@@ -89,7 +89,7 @@ def verify_user_password(user_id):
         return jsonify(result='error', message={'password': ['Incorrect password']}), 400
 
 
-@user.route('/<int:user_id>/verify/code', methods=['POST'])
+@user.route('/<uuid:user_id>/verify/code', methods=['POST'])
 def verify_user_code(user_id):
     user_to_verify = get_model_users(user_id=user_id)
 
@@ -116,7 +116,7 @@ def verify_user_code(user_id):
     return jsonify({}), 204
 
 
-@user.route('/<int:user_id>/sms-code', methods=['POST'])
+@user.route('/<uuid:user_id>/sms-code', methods=['POST'])
 def send_user_sms_code(user_id):
     user_to_send_to = get_model_users(user_id=user_id)
     verify_code, errors = request_verify_code_schema.load(request.get_json())
@@ -135,7 +135,7 @@ def send_user_sms_code(user_id):
     return jsonify({}), 204
 
 
-@user.route('/<int:user_id>/email-code', methods=['POST'])
+@user.route('/<uuid:user_id>/email-code', methods=['POST'])
 def send_user_email_code(user_id):
     user_to_send_to = get_model_users(user_id=user_id)
     verify_code, errors = request_verify_code_schema.load(request.get_json())
@@ -154,7 +154,7 @@ def send_user_email_code(user_id):
     return jsonify({}), 204
 
 
-@user.route('/<int:user_id>/email-verification', methods=['POST'])
+@user.route('/<uuid:user_id>/email-verification', methods=['POST'])
 def send_user_email_verification(user_id):
     user_to_send_to = get_model_users(user_id=user_id)
     verify_code, errors = request_verify_code_schema.load(request.get_json())
@@ -176,7 +176,7 @@ def send_user_email_verification(user_id):
     return jsonify({}), 204
 
 
-@user.route('/<int:user_id>', methods=['GET'])
+@user.route('/<uuid:user_id>', methods=['GET'])
 @user.route('', methods=['GET'])
 def get_user(user_id=None):
     users = get_model_users(user_id=user_id)
@@ -184,7 +184,7 @@ def get_user(user_id=None):
     return jsonify(data=result.data)
 
 
-@user.route('/<int:user_id>/service/<uuid:service_id>/permission', methods=['POST'])
+@user.route('/<uuid:user_id>/service/<uuid:service_id>/permission', methods=['POST'])
 def set_permissions(user_id, service_id):
     # TODO fix security hole, how do we verify that the user
     # who is making this request has permission to make the request.
@@ -240,7 +240,7 @@ def _create_reset_password_url(email):
 def _create_verification_url(user, secret_code):
     from utils.url_safe_token import generate_token
     import json
-    data = json.dumps({'user_id': user.id, 'email': user.email_address, 'secret_code': secret_code})
+    data = json.dumps({'user_id': str(user.id), 'email': user.email_address, 'secret_code': secret_code})
     token = generate_token(data, current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT'])
 
     return current_app.config['ADMIN_BASE_URL'] + '/verify-email/' + token