Merge branch 'master' into update-job-status

Conflicts:
	app/dao/jobs_dao.py
	tests/app/dao/test_jobs_dao.py

app/celery/research_mode_tasks.py

@@ -55,7 +55,8 @@ def make_request(notification_type, provider, data, headers):
             "POST",
             api_call,
             headers=headers,
-            data=data
+            data=data,
+            timeout=60
         )
         response.raise_for_status()
     except RequestException as e:

app/celery/scheduled_tasks.py

@@ -10,7 +10,7 @@ from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
-from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than
+from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
 from app.dao.notifications_dao import (
     delete_notifications_created_more_than_a_week_ago,
     dao_timeout_notifications,
@@ -28,7 +28,7 @@ from app.celery.tasks import process_job
 @notify_celery.task(name="remove_csv_files")
 @statsd(namespace="tasks")
 def remove_csv_files():
-    jobs = dao_get_jobs_older_than(7)
+    jobs = dao_get_jobs_older_than_limited_by()
     for job in jobs:
         s3.remove_job_from_s3(job.service_id, job.id)
         current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))

app/clients/sms/firetext.py

@@ -42,8 +42,10 @@ def get_firetext_responses(status):
 
 class FiretextClientResponseException(SmsClientResponseException):
     def __init__(self, response, exception):
-        self.status_code = response.status_code
-        self.text = response.text
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
         self.exception = exception
 
     def __str__(self):
@@ -68,11 +70,13 @@ class FiretextClient(SmsClient):
         return self.name
 
     def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
         log_message = "API {} request {} on {} response status_code {}".format(
             "POST",
             "succeeded" if success else "failed",
             self.url,
-            response.status_code
+            status_code
         )
 
         if success:
@@ -97,7 +101,8 @@ class FiretextClient(SmsClient):
             response = request(
                 "POST",
                 self.url,
-                data=data
+                data=data,
+                timeout=60
             )
             response.raise_for_status()
             try:

app/clients/sms/mmg.py

@@ -1,7 +1,6 @@
 import json
 from monotonic import monotonic
 from requests import (request, RequestException)
-
 from app.clients import (STATISTICS_DELIVERED, STATISTICS_FAILURE)
 from app.clients.sms import (SmsClient, SmsClientResponseException)
 
@@ -45,8 +44,11 @@ def get_mmg_responses(status):
 
 class MMGClientResponseException(SmsClientResponseException):
     def __init__(self, response, exception):
-        self.status_code = response.status_code
-        self.text = response.text
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
         self.exception = exception
 
     def __str__(self):
@@ -68,11 +70,12 @@ class MMGClient(SmsClient):
         self.mmg_url = current_app.config.get('MMG_URL')
 
     def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
         log_message = "API {} request {} on {} response status_code {}".format(
             "POST",
             "succeeded" if success else "failed",
             self.mmg_url,
-            response.status_code
+            status_code
         )
 
         if success:
@@ -104,7 +107,8 @@ class MMGClient(SmsClient):
                 headers={
                     'Content-Type': 'application/json',
                     'Authorization': 'Basic {}'.format(self.api_key)
-                }
+                },
+                timeout=60
             )
 
             response.raise_for_status()

app/dao/jobs_dao.py

@@ -121,7 +121,8 @@ def dao_update_job_status(job_id, status):
     db.session.commit()
 
 
-def dao_get_jobs_older_than(limit_days):
+def dao_get_jobs_older_than_limited_by(older_than=7, limit_days=2):
     return Job.query.filter(
-        cast(Job.created_at, sql_date) < days_ago(limit_days)
+        cast(Job.created_at, sql_date) < days_ago(older_than),
+        cast(Job.created_at, sql_date) >= days_ago(older_than + limit_days)
     ).order_by(desc(Job.created_at)).all()