Merge branch 'master' into remove-nasty-query-from-dashboard

app/celery/scheduled_tasks.py

@@ -9,6 +9,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
+from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
 from app.dao.notifications_dao import (
@@ -226,3 +227,21 @@ def timeout_job_statistics():
     if updated:
         current_app.logger.info(
             "Timeout period reached for {} job statistics, failure count has been updated.".format(updated))
+
+
+@notify_celery.task(name="delete-inbound-sms")
+@statsd(namespace="tasks")
+def delete_inbound_sms_older_than_seven_days():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_inbound_sms_created_more_than_a_week_ago()
+        current_app.logger.info(
+            "Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format(
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError as e:
+        current_app.logger.exception("Failed to delete inbound sms notifications")
+        raise

app/config.py

@@ -2,6 +2,7 @@ from datetime import timedelta
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 import os
+
 from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
 
 if os.environ.get('VCAP_SERVICES'):
@@ -168,6 +169,11 @@ class Config(object):
             'schedule': crontab(minute=40, hour=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(minute=0, hour=1),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
         'send-daily-performance-platform-stats': {
             'task': 'send-daily-performance-platform-stats',
             'schedule': crontab(minute=0, hour=2),

app/dao/inbound_sms_dao.py

@@ -1,6 +1,13 @@
+from datetime import (
+    timedelta,
+    datetime
+)
+
+
 from app import db
 from app.dao.dao_utils import transactional
 from app.models import InboundSms
+from app.statsd_decorators import statsd
 
 
 @transactional
@@ -28,3 +35,15 @@ def dao_count_inbound_sms_for_service(service_id):
     return InboundSms.query.filter(
         InboundSms.service_id == service_id
     ).count()
+
+
+@statsd(namespace="dao")
+@transactional
+def delete_inbound_sms_created_more_than_a_week_ago():
+    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+
+    deleted = db.session.query(InboundSms).filter(
+        InboundSms.created_at < seven_days_ago
+    ).delete(synchronize_session='fetch')
+
+    return deleted

app/dao/notifications_dao.py

@@ -2,7 +2,8 @@ import functools
 from datetime import (
     datetime,
     timedelta,
-    date)
+    date
+)
 
 from flask import current_app
 
@@ -26,6 +27,7 @@ from app.models import (
     NotificationHistory,
     NotificationStatistics,
     Template,
+    ScheduledNotification,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_SENDING,
@@ -35,7 +37,8 @@ from app.models import (
     NOTIFICATION_PERMANENT_FAILURE,
     KEY_TYPE_NORMAL, KEY_TYPE_TEST,
     LETTER_TYPE,
-    NOTIFICATION_SENT, ScheduledNotification)
+    NOTIFICATION_SENT,
+)
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd

app/models.py

@@ -1165,6 +1165,7 @@ class InboundSms(db.Model):
     user_number = db.Column(db.String, nullable=False)  # the end user's number, that the msg was sent from
     provider_date = db.Column(db.DateTime)
     provider_reference = db.Column(db.String)
+    provider = db.Column(db.String, nullable=True)
     _content = db.Column('content', db.String, nullable=False)
 
     @property

app/notifications/receive_notifications.py

@@ -4,7 +4,7 @@ import iso8601
 from flask import jsonify, Blueprint, current_app, request
 from notifications_utils.recipients import validate_and_format_phone_number
 
-from app import statsd_client
+from app import statsd_client, firetext_client, mmg_client
 from app.dao.services_dao import dao_fetch_services_by_sms_sender
 from app.dao.inbound_sms_dao import dao_create_inbound_sms
 from app.models import InboundSms
@@ -27,10 +27,13 @@ def receive_mmg_sms():
     }
     """
     post_data = request.get_json()
-    potential_services = dao_fetch_services_by_sms_sender(post_data['Number'])
+
+    inbound_number = strip_leading_forty_four(post_data['Number'])
+
+    potential_services = dao_fetch_services_by_sms_sender(inbound_number)
 
     if len(potential_services) != 1:
-        current_app.logger.error('Inbound number "{}" not associated with exactly one service'.format(
+        current_app.logger.error('Inbound number "{}" from MMG not associated with exactly one service'.format(
             post_data['Number']
         ))
         statsd_client.incr('inbound.mmg.failed')
@@ -38,7 +41,7 @@ def receive_mmg_sms():
         # succesfully
         return 'RECEIVED', 200
 
-    statsd_client.incr('inbound.mmg.succesful')
+    statsd_client.incr('inbound.mmg.successful')
 
     service = potential_services[0]
 
@@ -78,6 +81,7 @@ def create_inbound_mmg_sms_object(service, json):
         provider_date=provider_date,
         provider_reference=json.get('ID'),
         content=message,
+        provider=mmg_client.name
     )
     dao_create_inbound_sms(inbound)
     return inbound
@@ -86,8 +90,44 @@ def create_inbound_mmg_sms_object(service, json):
 @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
 def receive_firetext_sms():
     post_data = request.form
-    current_app.logger.info("Received Firetext notification form data: {}".format(post_data))
+
+    inbound_number = strip_leading_forty_four(post_data['destination'])
+
+    potential_services = dao_fetch_services_by_sms_sender(inbound_number)
+    if len(potential_services) != 1:
+        current_app.logger.error('Inbound number "{}" from firetext not associated with exactly one service'.format(
+            post_data['destination']
+        ))
+        statsd_client.incr('inbound.firetext.failed')
+        return jsonify({
+            "status": "ok"
+        }), 200
+
+    service = potential_services[0]
+
+    user_number = validate_and_format_phone_number(post_data['source'], international=True)
+    message = post_data['message']
+    timestamp = post_data['time']
+
+    dao_create_inbound_sms(
+        InboundSms(
+            service=service,
+            notify_number=service.sms_sender,
+            user_number=user_number,
+            provider_date=timestamp,
+            content=message,
+            provider=firetext_client.name
+        )
+    )
+
+    statsd_client.incr('inbound.firetext.successful')
 
     return jsonify({
         "status": "ok"
     }), 200
+
+
+def strip_leading_forty_four(number):
+    if number.startswith('44'):
+        return number.replace('44', '0', 1)
+    return number