make inbound sms honour data retention

code inspired by the delete notification code, but with some clean up
since we don't deal with different types etc, and only need to run the
query for services with inbound numbers

also, update tests.app.db.create_inbound_sms to create inbound numbers
and assign them to services to ensure the test db is always accurate
and reflects real world usage

app/dao/inbound_sms_dao.py

@@ -1,8 +1,3 @@
-from datetime import (
-    timedelta,
-    datetime,
-    date
-)
 from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import desc, and_
@@ -10,8 +5,8 @@ from sqlalchemy.orm import aliased
 
 from app import db
 from app.dao.dao_utils import transactional
-from app.models import InboundSms
-from app.utils import get_london_midnight_in_utc
+from app.models import InboundSms, Service, ServiceDataRetention, SMS_TYPE
+from app.utils import midnight_n_days_ago
 
 
 @transactional
@@ -20,7 +15,7 @@ def dao_create_inbound_sms(inbound_sms):
 
 
 def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     q = InboundSms.query.filter(
         InboundSms.service_id == service_id,
         InboundSms.created_at >= start_date
@@ -60,21 +55,63 @@ def dao_get_paginated_inbound_sms_for_service_for_public_api(
 
 
 def dao_count_inbound_sms_for_service(service_id):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     return InboundSms.query.filter(
         InboundSms.service_id == service_id,
         InboundSms.created_at >= start_date
     ).count()
 
 
+def _delete_inbound_sms(datetime_to_delete_from, query_filter):
+    query_limit = 10000
+
+    subquery = db.session.query(
+        InboundSms.id
+    ).filter(
+        InboundSms.created_at < datetime_to_delete_from,
+        *query_filter
+    ).limit(
+        query_limit
+    ).subquery()
+
+    deleted = 0
+    # set to nonzero just to enter the loop
+    number_deleted = 1
+    while number_deleted > 0:
+        number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(synchronize_session='fetch')
+        deleted += number_deleted
+
+    return deleted
+
+
 @statsd(namespace="dao")
 @transactional
-def delete_inbound_sms_created_more_than_a_week_ago():
-    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+def delete_inbound_sms_older_than_retention():
+    current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 
-    deleted = db.session.query(InboundSms).filter(
-        InboundSms.created_at < seven_days_ago
-    ).delete(synchronize_session='fetch')
+    flexible_data_retention = ServiceDataRetention.query.join(
+        ServiceDataRetention.service,
+        Service.inbound_number
+    ).filter(
+        ServiceDataRetention.notification_type == SMS_TYPE
+    ).all()
+
+    deleted = 0
+    for f in flexible_data_retention:
+        n_days_ago = midnight_n_days_ago(f.days_of_retention)
+
+        current_app.logger.info("Deleting inbound sms for service id: {}".format(f.service_id))
+        deleted += _delete_inbound_sms(n_days_ago, query_filter=[InboundSms.service_id == f.service_id])
+
+    current_app.logger.info('Deleting inbound sms for services without flexible data retention')
+
+    seven_days_ago = midnight_n_days_ago(7)
+
+    deleted += _delete_inbound_sms(seven_days_ago, query_filter=[
+        InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
+    ])
+
+    current_app.logger.info('Deleted {} inbound sms'.format(deleted))
 
     return deleted
 
@@ -108,7 +145,7 @@ def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
     ORDER BY t1.created_at DESC;
     LIMIT 50 OFFSET :page
     """
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     t2 = aliased(InboundSms)
     q = db.session.query(
         InboundSms