Merge branch 'master' into remove-fk-to-users

app/celery/nightly_tasks.py

@@ -16,14 +16,14 @@ from app.celery.service_callback_tasks import (
     create_delivery_status_callback_data,
 )
 from app.config import QueueNames
-from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
+from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
     dao_get_jobs_older_than_data_retention,
     dao_archive_job
 )
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
-    delete_notifications_created_more_than_a_week_ago_by_type,
+    delete_notifications_older_than_retention_by_type,
 )
 from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.exceptions import NotificationTechnicalFailureException
@@ -64,10 +64,10 @@ def _remove_csv_files(job_types):
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
 @statsd(namespace="tasks")
-def delete_sms_notifications_older_than_seven_days():
+def delete_sms_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago_by_type('sms')
+        deleted = delete_notifications_older_than_retention_by_type('sms')
         current_app.logger.info(
             "Delete {} job started {} finished {} deleted {} sms notifications".format(
                 'sms',
@@ -84,10 +84,10 @@ def delete_sms_notifications_older_than_seven_days():
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
 @statsd(namespace="tasks")
-def delete_email_notifications_older_than_seven_days():
+def delete_email_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago_by_type('email')
+        deleted = delete_notifications_older_than_retention_by_type('email')
         current_app.logger.info(
             "Delete {} job started {} finished {} deleted {} email notifications".format(
                 'email',
@@ -104,10 +104,10 @@ def delete_email_notifications_older_than_seven_days():
 @notify_celery.task(name="delete-letter-notifications")
 @cronitor("delete-letter-notifications")
 @statsd(namespace="tasks")
-def delete_letter_notifications_older_than_seven_days():
+def delete_letter_notifications_older_than_retention():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago_by_type('letter')
+        deleted = delete_notifications_older_than_retention_by_type('letter')
         current_app.logger.info(
             "Delete {} job started {} finished {} deleted {} letter notifications".format(
                 'letter',
@@ -190,10 +190,10 @@ def send_total_sent_notifications_to_performance_platform(day):
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
 @statsd(namespace="tasks")
-def delete_inbound_sms_older_than_seven_days():
+def delete_inbound_sms():
     try:
         start = datetime.utcnow()
-        deleted = delete_inbound_sms_created_more_than_a_week_ago()
+        deleted = delete_inbound_sms_older_than_retention()
         current_app.logger.info(
             "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format(
                 start,

app/celery/process_ses_receipts_tasks.py

@@ -42,10 +42,10 @@ def process_ses_results(self, response):
             notification = notifications_dao.dao_get_notification_by_reference(reference)
         except NoResultFound:
             message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
-            if datetime.utcnow() - message_time < timedelta(minutes=10):
+            if datetime.utcnow() - message_time < timedelta(minutes=5):
                 self.retry(queue=QueueNames.RETRY)
-            elif datetime.utcnow() - message_time < timedelta(days=3):
-                current_app.logger.error(
+            else:
+                current_app.logger.warning(
                     "notification not found for reference: {} (update to {})".format(reference, notification_status)
                 )
             return

app/dao/inbound_sms_dao.py

@@ -1,8 +1,3 @@
-from datetime import (
-    timedelta,
-    datetime,
-    date
-)
 from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import desc, and_
@@ -10,8 +5,8 @@ from sqlalchemy.orm import aliased
 
 from app import db
 from app.dao.dao_utils import transactional
-from app.models import InboundSms
-from app.utils import get_london_midnight_in_utc
+from app.models import InboundSms, Service, ServiceDataRetention, SMS_TYPE
+from app.utils import midnight_n_days_ago
 
 
 @transactional
@@ -19,14 +14,15 @@ def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
 
-def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None, limit_days=7):
     q = InboundSms.query.filter(
-        InboundSms.service_id == service_id,
-        InboundSms.created_at >= start_date
+        InboundSms.service_id == service_id
     ).order_by(
         InboundSms.created_at.desc()
     )
+    if limit_days is not None:
+        start_date = midnight_n_days_ago(limit_days)
+        q = q.filter(InboundSms.created_at >= start_date)
 
     if user_number:
         q = q.filter(InboundSms.user_number == user_number)
@@ -60,21 +56,63 @@ def dao_get_paginated_inbound_sms_for_service_for_public_api(
 
 
 def dao_count_inbound_sms_for_service(service_id):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     return InboundSms.query.filter(
         InboundSms.service_id == service_id,
         InboundSms.created_at >= start_date
     ).count()
 
 
+def _delete_inbound_sms(datetime_to_delete_from, query_filter):
+    query_limit = 10000
+
+    subquery = db.session.query(
+        InboundSms.id
+    ).filter(
+        InboundSms.created_at < datetime_to_delete_from,
+        *query_filter
+    ).limit(
+        query_limit
+    ).subquery()
+
+    deleted = 0
+    # set to nonzero just to enter the loop
+    number_deleted = 1
+    while number_deleted > 0:
+        number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(synchronize_session='fetch')
+        deleted += number_deleted
+
+    return deleted
+
+
 @statsd(namespace="dao")
 @transactional
-def delete_inbound_sms_created_more_than_a_week_ago():
-    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+def delete_inbound_sms_older_than_retention():
+    current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 
-    deleted = db.session.query(InboundSms).filter(
-        InboundSms.created_at < seven_days_ago
-    ).delete(synchronize_session='fetch')
+    flexible_data_retention = ServiceDataRetention.query.join(
+        ServiceDataRetention.service,
+        Service.inbound_number
+    ).filter(
+        ServiceDataRetention.notification_type == SMS_TYPE
+    ).all()
+
+    deleted = 0
+    for f in flexible_data_retention:
+        n_days_ago = midnight_n_days_ago(f.days_of_retention)
+
+        current_app.logger.info("Deleting inbound sms for service id: {}".format(f.service_id))
+        deleted += _delete_inbound_sms(n_days_ago, query_filter=[InboundSms.service_id == f.service_id])
+
+    current_app.logger.info('Deleting inbound sms for services without flexible data retention')
+
+    seven_days_ago = midnight_n_days_ago(7)
+
+    deleted += _delete_inbound_sms(seven_days_ago, query_filter=[
+        InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
+    ])
+
+    current_app.logger.info('Deleted {} inbound sms'.format(deleted))
 
     return deleted
 
@@ -108,7 +146,7 @@ def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
     ORDER BY t1.created_at DESC;
     LIMIT 50 OFFSET :page
     """
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     t2 = aliased(InboundSms)
     q = db.session.query(
         InboundSms

app/dao/notifications_dao.py

@@ -293,7 +293,7 @@ def _filter_query(query, filter_dict=None):
 
 
 @statsd(namespace="dao")
-def delete_notifications_created_more_than_a_week_ago_by_type(notification_type, qry_limit=10000):
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=10000):
     current_app.logger.info(
         'Deleting {} notifications for services with flexible data retention'.format(notification_type))
 

app/dao/organisation_dao.py

@@ -1,7 +1,10 @@
+from sqlalchemy.sql.expression import func
+
 from app import db
 from app.dao.dao_utils import transactional
 from app.models import (
     Organisation,
+    Domain,
     InvitedOrganisationUser,
     User
 )
@@ -23,6 +26,21 @@ def dao_get_organisation_by_id(organisation_id):
     return Organisation.query.filter_by(id=organisation_id).one()
 
 
+def dao_get_organisation_by_email_address(email_address):
+
+    email_address = email_address.lower()
+
+    for domain in Domain.query.order_by(func.char_length(Domain.domain).desc()).all():
+
+        if (
+            email_address.endswith("@{}".format(domain.domain)) or
+            email_address.endswith(".{}".format(domain.domain))
+        ):
+            return Organisation.query.filter_by(id=domain.organisation_id).one()
+
+    return None
+
+
 def dao_get_organisation_by_service_id(service_id):
     return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()
 
@@ -34,10 +52,26 @@ def dao_create_organisation(organisation):
 
 @transactional
 def dao_update_organisation(organisation_id, **kwargs):
-    return Organisation.query.filter_by(id=organisation_id).update(
+
+    domains = kwargs.pop('domains', [])
+
+    organisation = Organisation.query.filter_by(id=organisation_id).update(
         kwargs
     )
 
+    if isinstance(domains, list):
+
+        Domain.query.filter_by(organisation_id=organisation_id).delete()
+
+        db.session.bulk_save_objects([
+            Domain(domain=domain.lower(), organisation_id=organisation_id)
+            for domain in domains
+        ])
+
+        db.session.commit()
+
+    return organisation
+
 
 @transactional
 def dao_add_service_to_organisation(service, organisation_id):

app/dao/services_dao.py

@@ -11,6 +11,7 @@ from app.dao.dao_utils import (
     transactional,
     version_class
 )
+from app.dao.organisation_dao import dao_get_organisation_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.models import (
@@ -151,14 +152,25 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
 
 @transactional
 @version_class(Service)
-def dao_create_service(service, user, service_id=None, service_permissions=None, letter_branding=None):
+def dao_create_service(
+    service,
+    user,
+    service_id=None,
+    service_permissions=None,
+    letter_branding=None,
+):
     # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
     # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
     # migration is completed, this code should be able to be removed.
 
+    if not user:
+        raise ValueError("Can't create a service without a user")
+
     if service_permissions is None:
         service_permissions = DEFAULT_SERVICE_PERMISSIONS
 
+    organisation = dao_get_organisation_by_email_address(user.email_address)
+
     from app.dao.permissions_dao import permission_dao
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)
@@ -173,8 +185,20 @@ def dao_create_service(service, user, service_id=None, service_permissions=None,
 
     # do we just add the default - or will we get a value from FE?
     insert_service_sms_sender(service, current_app.config['FROM_NUMBER'])
+
     if letter_branding:
         service.letter_branding = letter_branding
+
+    if organisation:
+
+        service.organisation = organisation
+
+        if organisation.email_branding_id:
+            service.email_branding = organisation.email_branding_id
+
+        if organisation.letter_branding_id and not service.letter_branding:
+            service.letter_branding = organisation.letter_branding_id
+
     db.session.add(service)
 
 

app/models.py

@@ -316,6 +316,12 @@ organisation_to_service = db.Table(
 )
 
 
+class Domain(db.Model):
+    __tablename__ = "domain"
+    domain = db.Column(db.String(255), primary_key=True)
+    organisation_id = db.Column('organisation_id', UUID(as_uuid=True), db.ForeignKey('organisation.id'), nullable=False)
+
+
 class Organisation(db.Model):
     __tablename__ = "organisation"
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=False)
@@ -329,15 +335,50 @@ class Organisation(db.Model):
         secondary='organisation_to_service',
         uselist=True)
 
+    agreement_signed = db.Column(db.Boolean, nullable=True)
+    agreement_signed_at = db.Column(db.DateTime, nullable=True)
+    agreement_signed_by_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('users.id'),
+        nullable=True,
+    )
+    agreement_signed_version = db.Column(db.Float, nullable=True)
+    crown = db.Column(db.Boolean, nullable=True)
+    organisation_type = db.Column(db.String(255), nullable=True)
+
+    domains = db.relationship(
+        'Domain',
+    )
+
+    email_branding = db.relationship('EmailBranding')
+    email_branding_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('email_branding.id'),
+        nullable=True,
+    )
+
+    letter_branding = db.relationship('LetterBranding')
+    letter_branding_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('letter_branding.id'),
+        nullable=True,
+    )
+
     def serialize(self):
-        serialized = {
+        return {
             "id": str(self.id),
             "name": self.name,
             "active": self.active,
+            "crown": self.crown,
+            "organisation_type": self.organisation_type,
+            "letter_branding_id": self.letter_branding_id,
+            "email_branding_id": self.email_branding_id,
+            "agreement_signed": self.agreement_signed,
+            "agreement_signed_at": self.agreement_signed_at,
+            "agreement_signed_by_id": self.agreement_signed_by_id,
+            "agreement_signed_version": self.agreement_signed_version,
         }
 
-        return serialized
-
 
 class Service(db.Model, Versioned):
     __tablename__ = 'services'