Merge branch 'master' into schedule-api-notification

Conflicts:
	app/celery/scheduled_tasks.py
	tests/app/celery/test_scheduled_tasks.py

app/celery/scheduled_tasks.py

@@ -12,9 +12,9 @@ from app import performance_platform_client
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
 from app.dao.notifications_dao import (
-    delete_notifications_created_more_than_a_week_ago,
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
+    delete_notifications_created_more_than_a_week_ago_by_type,
     dao_get_scheduled_notifications,
     set_scheduled_notification_to_processed)
 from app.dao.statistics_dao import dao_timeout_job_statistics
@@ -78,42 +78,60 @@ def delete_verify_codes():
         raise
 
 
-@notify_celery.task(name="delete-successful-notifications")
+@notify_celery.task(name="delete-sms-notifications")
 @statsd(namespace="tasks")
-def delete_successful_notifications():
+def delete_sms_notifications_older_than_seven_days():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago('delivered')
+        deleted = delete_notifications_created_more_than_a_week_ago_by_type('sms')
         current_app.logger.info(
-            "Delete job started {} finished {} deleted {} successful notifications".format(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
                 start,
                 datetime.utcnow(),
                 deleted
             )
         )
     except SQLAlchemyError as e:
-        current_app.logger.exception("Failed to delete successful notifications")
+        current_app.logger.exception("Failed to delete sms notifications")
         raise
 
 
-@notify_celery.task(name="delete-failed-notifications")
+@notify_celery.task(name="delete-email-notifications")
 @statsd(namespace="tasks")
-def delete_failed_notifications():
+def delete_email_notifications_older_than_seven_days():
     try:
         start = datetime.utcnow()
-        deleted = delete_notifications_created_more_than_a_week_ago('failed')
-        deleted += delete_notifications_created_more_than_a_week_ago('technical-failure')
-        deleted += delete_notifications_created_more_than_a_week_ago('temporary-failure')
-        deleted += delete_notifications_created_more_than_a_week_ago('permanent-failure')
+        deleted = delete_notifications_created_more_than_a_week_ago_by_type('email')
         current_app.logger.info(
-            "Delete job started {} finished {} deleted {} failed notifications".format(
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
                 start,
                 datetime.utcnow(),
                 deleted
             )
         )
     except SQLAlchemyError as e:
-        current_app.logger.exception("Failed to delete failed notifications")
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise
+
+
+@notify_celery.task(name="delete-letter-notifications")
+@statsd(namespace="tasks")
+def delete_letter_notifications_older_than_seven_days():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_created_more_than_a_week_ago_by_type('letter')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
+                'letter',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError as e:
+        current_app.logger.exception("Failed to delete sms notifications")
         raise
 
 

app/commands.py

@@ -2,7 +2,10 @@ import uuid
 from datetime import datetime
 from decimal import Decimal
 from flask.ext.script import Command, Manager, Option
-from app.models import (PROVIDERS, Service, User)
+
+
+from app import db
+from app.models import (PROVIDERS, Service, User, NotificationHistory)
 from app.dao.services_dao import (
     delete_service_and_all_associated_db_objects,
     dao_fetch_all_services_by_user
@@ -60,3 +63,29 @@ class PurgeFunctionalTestDataCommand(Command):
                     else:
                         delete_user_verify_codes(usr)
                         delete_model_user(usr)
+
+
+class CustomDbScript(Command):
+    def run(self):
+        self.update_notification_international_flag()
+
+    def update_notification_international_flag(self):
+        # 250,000 rows takes 30 seconds to update.
+        subq = "select id from notifications where international is null limit 250000"
+        update = "update notifications set international = False where id in ({})".format(subq)
+        result = db.session.execute(subq).fetchall()
+        while len(result) > 0:
+            db.session.execute(update)
+            print('commit 250000 updates at {}'.format(datetime.utcnow()))
+            db.session.commit()
+            result = db.session.execute(subq).fetchall()
+
+        # Now update notification_history
+        subq_history = "select id from notification_history where international is null limit 250000"
+        update_history = "update notification_history set international = False where id in ({})".format(subq_history)
+        result_history = db.session.execute(subq_history).fetchall()
+        while len(result_history) > 0:
+            db.session.execute(update_history)
+            print('commit 250000 updates at {}'.format(datetime.utcnow()))
+            db.session.commit()
+            result_history = db.session.execute(subq_history).fetchall()

app/config.py

@@ -124,14 +124,19 @@ class Config(object):
             'schedule': timedelta(minutes=66),
             'options': {'queue': 'periodic'}
         },
-        'delete-failed-notifications': {
-            'task': 'delete-failed-notifications',
+        'delete-sms-notifications': {
+            'task': 'delete-sms-notifications',
             'schedule': crontab(minute=0, hour=0),
             'options': {'queue': 'periodic'}
         },
-        'delete-successful-notifications': {
-            'task': 'delete-successful-notifications',
-            'schedule': crontab(minute=0, hour=1),
+        'delete-email-notifications': {
+            'task': 'delete-email-notifications',
+            'schedule': crontab(minute=20, hour=0),
+            'options': {'queue': 'periodic'}
+        },
+        'delete-letter-notifications': {
+            'task': 'delete-letter-notifications',
+            'schedule': crontab(minute=40, hour=0),
             'options': {'queue': 'periodic'}
         },
         'send-daily-performance-platform-stats': {

app/dao/notifications_dao.py

@@ -351,11 +351,11 @@ def _filter_query(query, filter_dict=None):
 
 
 @statsd(namespace="dao")
-def delete_notifications_created_more_than_a_week_ago(status):
+def delete_notifications_created_more_than_a_week_ago_by_type(notification_type):
     seven_days_ago = date.today() - timedelta(days=7)
     deleted = db.session.query(Notification).filter(
         func.date(Notification.created_at) < seven_days_ago,
-        Notification.status == status,
+        Notification.notification_type == notification_type,
     ).delete(synchronize_session='fetch')
     db.session.commit()
     return deleted