Merge branch 'main' of https://github.com/GSA/notifications-api into notify-260

app/aws/s3.py

@@ -65,3 +65,14 @@ def remove_job_from_s3(service_id, job_id):
 def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
     obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
     return obj.delete()
+
+
+def remove_csv_object(object_key):
+    obj = get_s3_object(
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
+        object_key,
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region']
+    )
+    return obj.delete()

app/celery/nightly_tasks.py

@@ -5,6 +5,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app import notify_celery
 from app.aws import s3
+from app.aws.s3 import remove_csv_object
 from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.config import QueueNames
 from app.cronitor import cronitor
@@ -13,6 +14,7 @@ from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
     dao_archive_job,
     dao_get_jobs_older_than_data_retention,
+    dao_get_unfinished_jobs,
 )
 from app.dao.notifications_dao import (
     dao_get_notifications_processing_time_stats,
@@ -41,6 +43,19 @@ def _remove_csv_files(job_types):
         current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))
 
 
+@notify_celery.task(name="cleanup-unfinished-jobs")
+def cleanup_unfinished_jobs():
+    now = datetime.utcnow()
+    jobs = dao_get_unfinished_jobs()
+    for job in jobs:
+        # The query already checks that the processing_finished time is null, so here we are saying
+        # if it started more than 4 hours ago, that's too long
+        acceptable_finish_time = job.processing_started + timedelta(minutes=5)
+        if now > acceptable_finish_time:
+            remove_csv_object(job.original_file_name)
+            dao_archive_job(job)
+
+
 @notify_celery.task(name="delete-notifications-older-than-retention")
 def delete_notifications_older_than_retention():
     delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
@@ -158,6 +173,7 @@ def delete_inbound_sms():
 @notify_celery.task(name='save-daily-notification-processing-time')
 @cronitor("save-daily-notification-processing-time")
 def save_daily_notification_processing_time(local_date=None):
+
     # local_date is a string in the format of "YYYY-MM-DD"
     if local_date is None:
         # if a date is not provided, we run against yesterdays data

app/celery/provider_tasks.py

@@ -11,7 +11,10 @@ from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
 from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
 from app.dao import notifications_dao
-from app.dao.notifications_dao import update_notification_status_by_id
+from app.dao.notifications_dao import (
+    insert_notification_history_delete_notifications_by_id,
+    update_notification_status_by_id,
+)
 from app.delivery import send_to_providers
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
@@ -40,6 +43,10 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
     update_notification_status_by_id(notification_id, status, provider_response=provider_response)
     current_app.logger.info(f"Updated notification {notification_id} with response '{provider_response}'")
 
+    if status == NOTIFICATION_SENT:
+        insert_notification_history_delete_notifications_by_id(notification_id)
+        current_app.logger.info(f"Archived notification {notification_id} that was successfully sent")
+
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
 def deliver_sms(self, notification_id):

app/celery/research_mode_tasks.py

@@ -5,6 +5,7 @@ from requests import HTTPError, request
 
 from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
+from app.dao.notifications_dao import get_notification_by_id
 from app.models import SMS_TYPE
 
 temp_fail = "2028675303"
@@ -16,8 +17,8 @@ perm_fail_email = "perm-fail@simulator.notify"
 temp_fail_email = "temp-fail@simulator.notify"
 
 
-def send_sms_response(provider, reference, to):
-    body = sns_callback(reference, to)
+def send_sms_response(provider, reference):
+    body = sns_callback(reference)
     headers = {"Content-type": "application/json"}
 
     make_request(SMS_TYPE, provider, body, headers)
@@ -59,25 +60,16 @@ def make_request(notification_type, provider, data, headers):
     return response.json()
 
 
-def sns_callback(notification_id, to):
-    raise Exception("Need to update for SNS callback format along with test_send_to_providers")
+def sns_callback(notification_id):
+    notification = get_notification_by_id(notification_id)
 
-    # example from mmg_callback
-    # if to.strip().endswith(temp_fail):
-    #     # status: 4 - expired (temp failure)
-    #     status = "4"
-    # elif to.strip().endswith(perm_fail):
-    #     # status: 5 - rejected (perm failure)
-    #     status = "5"
-    # else:
-    #     # status: 3 - delivered
-    #     status = "3"
-
-    # return json.dumps({"reference": "mmg_reference",
-    #                    "CID": str(notification_id),
-    #                    "MSISDN": to,
-    #                    "status": status,
-    #                    "deliverytime": "2016-04-05 16:01:07"})
+    # This will only work if all notifications, including successful ones, are in the notifications table
+    # If we decide to delete successful notifications, we will have to get this from notifications history
+    return json.dumps({
+        "CID": str(notification_id),
+        "status": notification.status,
+        # "deliverytime": notification.completed_at
+    })
 
 
 def ses_notification_callback(reference):

app/commands.py

@@ -18,6 +18,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app import db
 from app.aws import s3
+from app.celery.nightly_tasks import cleanup_unfinished_jobs
 from app.celery.tasks import process_row
 from app.dao.annual_billing_dao import (
     dao_create_or_update_annual_billing_for_year,
@@ -464,6 +465,12 @@ def fix_billable_units():
     print("End fix_billable_units")
 
 
+@notify_command(name='delete-unfinished-jobs')
+def delete_unfinished_jobs():
+    cleanup_unfinished_jobs()
+    print("End cleanup_unfinished_jobs")
+
+
 @notify_command(name='process-row-from-job')
 @click.option('-j', '--job_id', required=True, help='Job id')
 @click.option('-n', '--job_row_number', type=int, required=True, help='Job id')

app/config.py

@@ -240,6 +240,11 @@ class Config(object):
                 'schedule': crontab(hour=2, minute=0),
                 'options': {'queue': QueueNames.PERIODIC}
             },
+            'cleanup-unfinished-jobs': {
+                'task': 'cleanup-unfinished-jobs',
+                'schedule': crontab(hour=0, minute=5),
+                'options': {'queue': QueueNames.PERIODIC}
+            },
             'remove_sms_email_jobs': {
                 'task': 'remove_sms_email_jobs',
                 'schedule': crontab(hour=4, minute=0),

app/dao/jobs_dao.py

@@ -43,6 +43,10 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
     return Job.query.filter_by(service_id=service_id, id=job_id).one()
 
 
+def dao_get_unfinished_jobs():
+    return Job.query.filter(Job.processing_finished.is_(None)).all()
+
+
 def dao_get_jobs_by_service_id(
     service_id,
     *,

app/dao/notifications_dao.py

@@ -271,6 +271,37 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
+@autocommit
+def insert_notification_history_delete_notifications_by_id(
+    notification_id
+):
+    """
+    Deletes one notification after it has run successfully and moves it to the notification_history
+    table.
+    """
+    input_params = {
+        "notification_id": notification_id
+    }
+    # Insert into NotificationHistory if the row already exists do nothing.
+    insert_query = """
+        insert into notification_history
+         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
+             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
+             client_reference, international, phone_prefix, rate_multiplier, notification_status,
+              created_by_id, document_download_count
+          from NOTIFICATIONS WHERE id= :notification_id
+          ON CONFLICT ON CONSTRAINT notification_history_pkey
+          DO NOTHING
+    """
+    delete_query = """
+        DELETE FROM notifications
+        where id= :notification_id
+    """
+
+    db.session.execute(insert_query, input_params)
+    db.session.execute(delete_query, input_params)
+
+
 @autocommit
 def insert_notification_history_delete_notifications(
     notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000

app/delivery/send_to_providers.py

@@ -61,7 +61,7 @@ def send_sms_to_provider(notification):
         )
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.name, str(notification.id))
 
         else:
             try:

app/models.py

@@ -108,7 +108,7 @@ class User(db.Model):
     platform_admin = db.Column(db.Boolean, nullable=False, default=False)
     current_session_id = db.Column(UUID(as_uuid=True), nullable=True)
     auth_type = db.Column(
-        db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE
+        db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=SMS_AUTH_TYPE
     )
     email_access_validated_at = db.Column(
         db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow
@@ -1651,7 +1651,7 @@ class InvitedUser(db.Model):
         db.ForeignKey('auth_type.name'),
         index=True,
         nullable=False,
-        default=EMAIL_AUTH_TYPE
+        default=SMS_AUTH_TYPE
     )
     folder_permissions = db.Column(JSONB(none_as_null=True), nullable=False, default=[])