diff --git a/app/celery/tasks.py b/app/celery/tasks.py index a7a47e087..d4913e24d 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -15,6 +15,7 @@ from datetime import datetime @notify_celery.task(name="process-job") def process_job(job_id): + start = datetime.utcnow() job = dao_get_job_by_id(job_id) job.status = 'in progress' dao_update_job(job) @@ -33,7 +34,8 @@ def process_job(job_id): send_sms.apply_async(( str(job.service_id), str(create_uuid()), - encrypted), + encrypted, + str(datetime.utcnow())), queue='bulk-sms' ) @@ -43,11 +45,18 @@ def process_job(job_id): str(create_uuid()), job.template.subject, "{}@{}".format(job.service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']), - encrypted), + encrypted, + str(datetime.utcnow())), queue='bulk-email') + finished = datetime.utcnow() job.status = 'finished' + job.processing_started = start + job.processing_finished = finished dao_update_job(job) + current_app.logger.info( + "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished) + ) @notify_celery.task(name="send-sms") @@ -55,7 +64,10 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): notification = encryption.decrypt(encrypted_notification) template = dao_get_template_by_id(notification['template']) + client = firetext_client + try: + sent_at = datetime.utcnow() notification_db_object = Notification( id=notification_id, template_id=notification['template'], @@ -64,17 +76,22 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): job_id=notification.get('job', None), status='sent', created_at=created_at, - sent_at=datetime.utcnow() + sent_at=sent_at, + sent_by=client.get_name() + ) dao_create_notification(notification_db_object) try: - firetext_client.send_sms(notification['to'], template.content) + client.send_sms(notification['to'], template.content) except FiretextClientException as e: current_app.logger.debug(e) notification_db_object.status = 'failed' dao_update_notification(notification_db_object) + current_app.logger.info( + "SMS {} created at {} sent at {}".format(notification_id, created_at, sent_at) + ) except SQLAlchemyError as e: current_app.logger.debug(e) @@ -84,7 +101,10 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not notification = encryption.decrypt(encrypted_notification) template = dao_get_template_by_id(notification['template']) + client = aws_ses_client + try: + sent_at = datetime.utcnow() notification_db_object = Notification( id=notification_id, template_id=notification['template'], @@ -93,12 +113,13 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not job_id=notification.get('job', None), status='sent', created_at=created_at, - sent_at=datetime.utcnow() + sent_at=sent_at, + sent_by=client.get_name() ) dao_create_notification(notification_db_object) try: - aws_ses_client.send_email( + client.send_email( from_address, notification['to'], subject, @@ -109,6 +130,9 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not notification_db_object.status = 'failed' dao_update_notification(notification_db_object) + current_app.logger.info( + "Email {} created at {} sent at {}".format(notification_id, created_at, sent_at) + ) except SQLAlchemyError as e: current_app.logger.debug(e) diff --git a/app/clients/email/__init__.py b/app/clients/email/__init__.py index 98f0cb901..15f250496 100644 --- a/app/clients/email/__init__.py +++ b/app/clients/email/__init__.py @@ -15,3 +15,6 @@ class EmailClient(Client): def send_email(self, *args, **kwargs): raise NotImplemented('TODO Need to implement.') + + def get_name(self): + raise NotImplemented('TODO Need to implement.') diff --git a/app/clients/email/aws_ses.py b/app/clients/email/aws_ses.py index 133368d84..c1f355b43 100644 --- a/app/clients/email/aws_ses.py +++ b/app/clients/email/aws_ses.py @@ -15,6 +15,10 @@ class AwsSesClient(EmailClient): def init_app(self, region, *args, **kwargs): self._client = boto3.client('ses', region_name=region) super(AwsSesClient, self).__init__(*args, **kwargs) + self.name = 'ses' + + def get_name(self): + return self.name def send_email(self, source, diff --git a/app/clients/sms/__init__.py b/app/clients/sms/__init__.py index 8b83ab8b9..90d63ed0d 100644 --- a/app/clients/sms/__init__.py +++ b/app/clients/sms/__init__.py @@ -15,3 +15,6 @@ class SmsClient(Client): def send_sms(self, *args, **kwargs): raise NotImplemented('TODO Need to implement.') + + def get_name(self): + raise NotImplemented('TODO Need to implement.') diff --git a/app/clients/sms/firetext.py b/app/clients/sms/firetext.py index 4a70947a1..235880e83 100644 --- a/app/clients/sms/firetext.py +++ b/app/clients/sms/firetext.py @@ -21,6 +21,10 @@ class FiretextClient(SmsClient): super(SmsClient, self).__init__(*args, **kwargs) self.api_key = config.config.get('FIRETEXT_API_KEY') self.from_number = config.config.get('FIRETEXT_NUMBER') + self.name = 'firetext' + + def get_name(self): + return self.name def send_sms(self, to, content): diff --git a/app/clients/sms/twilio.py b/app/clients/sms/twilio.py index eb5613878..ca33960fb 100644 --- a/app/clients/sms/twilio.py +++ b/app/clients/sms/twilio.py @@ -22,6 +22,10 @@ class TwilioClient(SmsClient): config.config.get('TWILIO_ACCOUNT_SID'), config.config.get('TWILIO_AUTH_TOKEN')) self.from_number = config.config.get('TWILIO_NUMBER') + self.name = 'twilio' + + def get_name(self): + return self.name def send_sms(self, to, content): try: diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 035a10a7e..137305eb9 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -1,3 +1,5 @@ +from datetime import datetime + from flask import ( Blueprint, jsonify, @@ -115,7 +117,8 @@ def send_notification(notification_type, service_id=None, expects_job=False): send_sms.apply_async(( service_id, notification_id, - encryption.encrypt(notification)), + encryption.encrypt(notification), + str(datetime.utcnow())), queue='sms') else: send_email.apply_async(( @@ -123,6 +126,7 @@ def send_notification(notification_type, service_id=None, expects_job=False): notification_id, template.subject, "{}@{}".format(service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']), - encryption.encrypt(notification)), + encryption.encrypt(notification), + str(datetime.utcnow())), queue='email') return jsonify({'notification_id': notification_id}), 201 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 60ceea35b..dbbcc7ae3 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -4,4 +4,5 @@ pytest==2.8.1 pytest-mock==0.8.1 pytest-cov==2.2.0 mock==1.0.1 -moto==0.4.19 \ No newline at end of file +moto==0.4.19 +freezegun==0.3.6 diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index c0abb4c06..0c25826d5 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -12,8 +12,10 @@ from app.celery.tasks import s3 from app.celery import tasks from tests.app import load_example_csv from datetime import datetime +from freezegun import freeze_time +@freeze_time("2016-01-01 11:09:00.061258") def test_should_process_sms_job(sample_job, mocker): mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms')) mocker.patch('app.celery.tasks.send_sms.apply_async') @@ -26,7 +28,8 @@ def test_should_process_sms_job(sample_job, mocker): tasks.send_sms.apply_async.assert_called_once_with( (str(sample_job.service_id), "uuid", - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="bulk-sms" ) job = jobs_dao.dao_get_job_by_id(sample_job.id) @@ -45,6 +48,7 @@ def test_should_not_create_send_task_for_empty_file(sample_job, mocker): tasks.send_sms.apply_async.assert_not_called +@freeze_time("2016-01-01 11:09:00.061258") def test_should_process_email_job(sample_email_job, mocker): mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('email')) mocker.patch('app.celery.tasks.send_email.apply_async') @@ -59,7 +63,8 @@ def test_should_process_email_job(sample_email_job, mocker): "uuid", sample_email_job.template.subject, "{}@{}".format(sample_email_job.service.email_from, "test.notify.com"), - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="bulk-email" ) job = jobs_dao.dao_get_job_by_id(sample_email_job.id) @@ -87,6 +92,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms') + mocker.patch('app.firetext_client.get_name', return_value="firetext") notification_id = uuid.uuid4() now = datetime.utcnow() @@ -105,6 +111,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat assert persisted_notification.status == 'sent' assert persisted_notification.created_at == now assert persisted_notification.sent_at > now + assert persisted_notification.sent_by == 'firetext' assert not persisted_notification.job_id @@ -116,6 +123,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms') + mocker.patch('app.firetext_client.get_name', return_value="firetext") notification_id = uuid.uuid4() now = datetime.utcnow() @@ -134,6 +142,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa assert persisted_notification.status == 'sent' assert persisted_notification.sent_at > now assert persisted_notification.created_at == now + assert persisted_notification.sent_by == 'firetext' def test_should_use_email_template_and_persist(sample_email_template, mocker): @@ -143,6 +152,7 @@ def test_should_use_email_template_and_persist(sample_email_template, mocker): } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.get_name', return_value='ses') notification_id = uuid.uuid4() now = datetime.utcnow() @@ -167,6 +177,7 @@ def test_should_use_email_template_and_persist(sample_email_template, mocker): assert persisted_notification.created_at == now assert persisted_notification.sent_at > now assert persisted_notification.status == 'sent' + assert persisted_notification.sent_by == 'ses' def test_should_persist_notification_as_failed_if_sms_client_fails(sample_template, mocker): @@ -176,6 +187,7 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException()) + mocker.patch('app.firetext_client.get_name', return_value="firetext") now = datetime.utcnow() notification_id = uuid.uuid4() @@ -194,6 +206,7 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa assert persisted_notification.status == 'failed' assert persisted_notification.created_at == now assert persisted_notification.sent_at > now + assert persisted_notification.sent_by == 'firetext' def test_should_persist_notification_as_failed_if_email_client_fails(sample_email_template, mocker): @@ -203,6 +216,8 @@ def test_should_persist_notification_as_failed_if_email_client_fails(sample_emai } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.aws_ses_client.send_email', side_effect=AwsSesClientException()) + mocker.patch('app.aws_ses_client.get_name', return_value="ses") + now = datetime.utcnow() notification_id = uuid.uuid4() @@ -227,6 +242,7 @@ def test_should_persist_notification_as_failed_if_email_client_fails(sample_emai assert persisted_notification.template_id == sample_email_template.id assert persisted_notification.status == 'failed' assert persisted_notification.created_at == now + assert persisted_notification.sent_by == 'ses' assert persisted_notification.sent_at > now diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 504c7d0dc..776db5a3e 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -6,6 +6,7 @@ from app.models import Service from app.dao.templates_dao import dao_get_all_templates_for_service from app.dao.services_dao import dao_update_service from tests.app.conftest import sample_job +from freezegun import freeze_time def test_get_notification_by_id(notify_api, sample_notification): @@ -427,6 +428,7 @@ def test_should_not_allow_template_from_another_service_on_job_sms( assert test_string in json_resp['message']['template'] +@freeze_time("2016-01-01 11:09:00.061258") def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: @@ -454,13 +456,15 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker app.celery.tasks.send_sms.apply_async.assert_called_once_with( (str(sample_template.service_id), notification_id, - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="sms" ) assert response.status_code == 201 assert notification_id +@freeze_time("2016-01-01 11:09:00.061258") def test_should_allow_valid_sms_notification_for_job(notify_api, sample_job, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: @@ -489,7 +493,8 @@ def test_should_allow_valid_sms_notification_for_job(notify_api, sample_job, moc app.celery.tasks.send_sms.apply_async.assert_called_once_with( (str(sample_job.service_id), notification_id, - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="sms" ) assert response.status_code == 201 @@ -812,6 +817,7 @@ def test_should_not_send_email_for_job_if_restricted_and_not_a_service_user( assert 'Email address not permitted for restricted service' in json_resp['message']['to'] +@freeze_time("2016-01-01 11:09:00.061258") def test_should_allow_valid_email_notification(notify_api, sample_email_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: @@ -841,7 +847,8 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template notification_id, "Email Subject", "sample.service@test.notify.com", - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="email" ) assert response.status_code == 201 @@ -880,6 +887,7 @@ def test_send_notification_invalid_job_id_on_job_email(notify_api, sample_email_ assert test_string in json_resp['message']['job'] +@freeze_time("2016-01-01 11:09:00.061258") def test_should_allow_valid_email_notification_for_job(notify_api, sample_job, sample_email_template, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: @@ -904,14 +912,14 @@ def test_should_allow_valid_email_notification_for_job(notify_api, sample_job, s data=json.dumps(data), headers=[('Content-Type', 'application/json'), auth_header]) - print(json.loads(response.data)) notification_id = json.loads(response.data)['notification_id'] app.celery.tasks.send_email.apply_async.assert_called_once_with( (str(sample_job.service_id), notification_id, "Email Subject", "sample.service@test.notify.com", - "something_encrypted"), + "something_encrypted", + "2016-01-01 11:09:00.061258"), queue="email" ) assert response.status_code == 201