From 9617f0748b64e1f9fe88080e12eab728f14b2aed Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 31 May 2016 12:49:06 +0100 Subject: [PATCH 1/9] Added some tests around creating and updated services - ensure research mode is respected on creation and update - ensure rest client gives an error for bad research mode update --- app/dao/services_dao.py | 1 + tests/app/dao/test_services_dao.py | 1 + tests/app/service/test_rest.py | 61 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 1563e7fb1..fd73d03e4 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -48,6 +48,7 @@ def dao_create_service(service, user): service.users.append(user) permission_dao.add_default_service_permissions_for_user(user, service) service.id = uuid.uuid4() # must be set now so version history model can use same id + service.research_mode = False db.session.add(service) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 0e431d4e2..199535d27 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -42,6 +42,7 @@ def test_create_service(sample_user): assert Service.query.count() == 1 assert Service.query.first().name == "service_name" assert Service.query.first().id == service.id + assert not Service.query.first().research_mode assert sample_user in Service.query.first().users diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 845bcdf79..7c8b52c37 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -101,6 +101,7 @@ def test_get_service_by_id(notify_api, sample_service): json_resp = json.loads(resp.get_data(as_text=True)) assert json_resp['data']['name'] == sample_service.name assert json_resp['data']['id'] == str(sample_service.id) + assert not json_resp['data']['research_mode'] def test_get_service_by_id_should_404_if_no_service(notify_api, notify_db): @@ -170,6 +171,7 @@ def test_create_service(notify_api, sample_user): assert json_resp['data']['id'] assert json_resp['data']['name'] == 'created service' assert json_resp['data']['email_from'] == 'created.service' + assert not json_resp['data']['research_mode'] auth_header_fetch = create_authorization_header() @@ -180,6 +182,7 @@ def test_create_service(notify_api, sample_user): assert resp.status_code == 200 json_resp = json.loads(resp.get_data(as_text=True)) assert json_resp['data']['name'] == 'created service' + assert not json_resp['data']['research_mode'] def test_should_not_create_service_with_missing_user_id_field(notify_api, fake_uuid): @@ -361,6 +364,64 @@ def test_update_service(notify_api, sample_service): assert result['data']['email_from'] == 'updated.service.name' +def test_update_service_research_mode(notify_api, sample_service): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + auth_header = create_authorization_header() + resp = client.get( + '/service/{}'.format(sample_service.id), + headers=[auth_header] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 200 + assert json_resp['data']['name'] == sample_service.name + assert not json_resp['data']['research_mode'] + + data = { + 'research_mode': True + } + + auth_header = create_authorization_header() + + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 200 + assert result['data']['research_mode'] + + +def test_update_service_research_mode_throws_validation_error(notify_api, sample_service): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + auth_header = create_authorization_header() + resp = client.get( + '/service/{}'.format(sample_service.id), + headers=[auth_header] + ) + json_resp = json.loads(resp.get_data(as_text=True)) + assert resp.status_code == 200 + assert json_resp['data']['name'] == sample_service.name + assert not json_resp['data']['research_mode'] + + data = { + 'research_mode': "dedede" + } + + auth_header = create_authorization_header() + + resp = client.post( + '/service/{}'.format(sample_service.id), + data=json.dumps(data), + headers=[('Content-Type', 'application/json'), auth_header] + ) + result = json.loads(resp.get_data(as_text=True)) + result['message']['research_mode'][0] == "Not a valid boolean." + assert resp.status_code == 400 + + def test_should_not_update_service_with_duplicate_name(notify_api, notify_db, notify_db_session, From 909fac3c05ac2028e1f0b29ec34bf4a5315a4fe8 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Tue, 31 May 2016 16:55:26 +0100 Subject: [PATCH 2/9] Added research mode tasks - if a service is in research mode the don't send the notifications via the providers (MMG/SES/etc) - instead set up a task to mimic those services callbacks - this completes the loop, and show stats, delivery receipts and so on. - Use the "to" field to choose the response, allows users to create successful and errored notifications temp fail sms, uses "07833333333" perm fail sms, uses = "07822222222" success = "07811111111" (or anything else) success email = "delivered@simulator.notify" perm fail = "perm-fail@simulator.notify" temp fail = "temp-fail@simulator.notify" --- app/celery/research_mode_tasks.py | 107 +++++++++++++++++++ app/celery/tasks.py | 36 ++++--- app/notifications/rest.py | 1 - config.py | 3 + environment_test.sh | 1 + requirements_for_test.txt | 1 + tests/app/celery/test_research_mode_tasks.py | 90 ++++++++++++++++ tests/app/celery/test_tasks.py | 84 +++++++++++++++ tests/app/conftest.py | 7 ++ 9 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 app/celery/research_mode_tasks.py create mode 100644 tests/app/celery/test_research_mode_tasks.py diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py new file mode 100644 index 000000000..0e76d5158 --- /dev/null +++ b/app/celery/research_mode_tasks.py @@ -0,0 +1,107 @@ +import json + +from flask import current_app +from app import notify_celery +from requests import request, RequestException, HTTPError + +temp_fail = "07833333333" +perm_fail = "07822222222" +delivered = "07811111111" + +delivered_email = "delivered@simulator.notify" +perm_fail_email = "perm-fail@simulator.notify" +temp_fail_email = "temp-fail@simulator.notify" + + +@notify_celery.task(name="send-mmg-response") +def send_sms_response(provider, reference, to): + if provider == "mmg": + body = mmg_callback(reference, to) + headers = {"Content-type": "application/json"} + if provider == "firetext": + headers = {"Content-type": "text/plain"} + body = firetext_callback(reference, to) + make_request('sms', provider, body, headers) + + +@notify_celery.task(name="send-ses-response") +def send_email_response(provider, reference, to): + if to == perm_fail_email: + body = ses_hard_bounce_callback(reference) + elif to == temp_fail_email: + body = ses_soft_bounce_callback(reference) + else: + body = ses_notification_callback(reference) + + make_request('email', provider, body, headers={"Content-type": "application/json"}) + + +def make_request(notification_type, provider, data, headers): + api_call = "{}/notifications/{}/{}".format(current_app.config["API_HOST_NAME"], notification_type, provider) + + try: + response = request( + "POST", + api_call, + headers=headers, + data=data + ) + response.raise_for_status() + except RequestException as e: + api_error = HTTPError(e) + current_app.logger.error( + "API {} request on {} failed with {}".format( + "POST", + api_call, + api_error.response + ) + ) + raise api_error + finally: + current_app.logger.info("Mocked provider callback request finished") + return response.json() + + +def mmg_callback(notification_id, to): + """ + status: 3 - delivered + status: 4 - expired (temp failure) + status: 5 - rejected (perm failure) + """ + + if to == temp_fail: + status = "4" + elif to == perm_fail: + status = "5" + else: + status = "3" + + return json.dumps({"reference": "mmg_reference", + "CID": str(notification_id), + "MSISDN": to, + "status": status, + "deliverytime": "2016-04-05 16:01:07"}) + + +def firetext_callback(notification_id, to): + """ + status: 0 - delivered + status: 1 - perm failure + """ + if to == perm_fail: + status = "1" + else: + status = "0" + return 'mobile={}&status={}&time=2016-03-10 14:17:00&reference={}'.format(to, status, notification_id) + + +def ses_notification_callback(reference): + return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa + + +def ses_hard_bounce_callback(reference): + return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa + + +def ses_soft_bounce_callback(reference): + return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa diff --git a/app/celery/tasks.py b/app/celery/tasks.py index cf6c9dcdc..e68169fc5 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -10,6 +10,7 @@ from app.clients.sms import SmsClientException from app.dao.services_dao import dao_fetch_service_by_id from app.dao.templates_dao import dao_get_template_by_id from app.dao.provider_details_dao import get_provider_details_by_notification_type +from app.celery.research_mode_tasks import send_email_response, send_sms_response from notifications_utils.template import Template, unlink_govuk_escaped @@ -261,11 +262,14 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): return try: - provider.send_sms( - to=validate_and_format_phone_number(notification['to']), - content=template.replaced, - reference=str(notification_id) - ) + if service.research_mode: + send_sms_response.apply_async((provider.get_name(), str(notification_id), notification['to']), queue='sms') + else: + provider.send_sms( + to=validate_and_format_phone_number(notification['to']), + content=template.replaced, + reference=str(notification_id) + ) except SmsClientException as e: current_app.logger.error( @@ -332,14 +336,18 @@ def send_email(service_id, notification_id, from_address, encrypted_notification values=notification.get('personalisation', {}) ) - reference = provider.send_email( - from_address, - notification['to'], - template.replaced_subject, - body=template.replaced_govuk_escaped, - html_body=template.as_HTML_email, - reply_to_addresses=reply_to_addresses, - ) + if service.research_mode: + reference = create_uuid() + send_email_response.apply_async((provider.get_name(), str(reference), notification['to']), queue='email') + else: + reference = provider.send_email( + from_address, + notification['to'], + template.replaced_subject, + body=template.replaced_govuk_escaped, + html_body=template.as_HTML_email, + reply_to_addresses=reply_to_addresses, + ) update_notification_reference_by_id(notification_id, reference) @@ -500,7 +508,7 @@ def service_allowed_to_send_to(recipient, service): def provider_to_use(notification_type, notification_id): active_providers_in_order = [ provider for provider in get_provider_details_by_notification_type(notification_type) if provider.active - ] + ] if not active_providers_in_order: current_app.logger.error( diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 8729b1058..d87147dac 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -46,7 +46,6 @@ def process_ses_response(): client_name = 'SES' try: ses_request = json.loads(request.data) - errors = validate_callback_data(data=ses_request, fields=['Message'], client_name=client_name) if errors: return jsonify( diff --git a/config.py b/config.py index b1dd2d22c..f3cc20aca 100644 --- a/config.py +++ b/config.py @@ -95,17 +95,20 @@ class Config(object): class Development(Config): DEBUG = True + API_HOST_NAME = os.environ['API_HOST_NAME'] MMG_API_KEY = os.environ['MMG_API_KEY'] CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' class Preview(Config): MMG_API_KEY = os.environ['MMG_API_KEY'] + API_HOST_NAME = os.environ['API_HOST_NAME'] CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' class Test(Development): MMG_API_KEY = os.environ['MMG_API_KEY'] + API_HOST_NAME = os.environ['API_HOST_NAME'] CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' diff --git a/environment_test.sh b/environment_test.sh index 137ce3ff5..7c99e5d87 100644 --- a/environment_test.sh +++ b/environment_test.sh @@ -26,3 +26,4 @@ export STATSD_ENABLED=True export STATSD_HOST="localhost" export STATSD_PORT=1000 export STATSD_PREFIX="stats-prefix" +export API_HOST_NAME="http://localhost:6011" diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 66f6f39eb..3e0cf151d 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -6,3 +6,4 @@ pytest-cov==2.2.0 mock==1.0.1 moto==0.4.19 freezegun==0.3.6 +requests-mock==0.7.0 diff --git a/tests/app/celery/test_research_mode_tasks.py b/tests/app/celery/test_research_mode_tasks.py new file mode 100644 index 000000000..c82f8b955 --- /dev/null +++ b/tests/app/celery/test_research_mode_tasks.py @@ -0,0 +1,90 @@ +from flask import json +from app.celery.research_mode_tasks import ( + send_sms_response, + send_email_response, + mmg_callback, + firetext_callback, + ses_notification_callback, + ses_hard_bounce_callback, + ses_soft_bounce_callback +) + + +def test_make_mmg_callback(notify_api, rmock): + endpoint = "http://localhost:6011/notifications/sms/mmg" + rmock.request( + "POST", + endpoint, + json={"status": "success"}, + status_code=200) + send_sms_response("mmg", "1234", "07811111111") + + assert rmock.called + + +def test_make_firetext_callback(notify_api, rmock): + endpoint = "http://localhost:6011/notifications/sms/firetext" + rmock.request( + "POST", + endpoint, + data="some data", + status_code=200) + send_sms_response("firetext", "1234", "07811111111") + + assert rmock.called + + +def test_make_ses_callback(notify_api, rmock): + endpoint = "http://localhost:6011/notifications/email/ses" + rmock.request( + "POST", + endpoint, + json={"status": "success"}, + status_code=200) + send_email_response("ses", "1234", "test@test.com") + + assert rmock.called + + +def test_delivered_mmg_callback(): + data = json.loads(mmg_callback("1234", "07811111111")) + assert data['MSISDN'] == "07811111111" + assert data['status'] == "0" + assert data['reference'] == "mmg_reference" + assert data['CID'] == "1234" + + +def test_perm_failure_mmg_callback(): + data = json.loads(mmg_callback("1234", "07822222222")) + assert data['MSISDN'] == "07822222222" + assert data['status'] == "5" + assert data['reference'] == "mmg_reference" + assert data['CID'] == "1234" + + +def test_temp_failure_mmg_callback(): + data = json.loads(mmg_callback("1234", "07833333333")) + assert data['MSISDN'] == "07833333333" + assert data['status'] == "4" + assert data['reference'] == "mmg_reference" + assert data['CID'] == "1234" + + +def test_delivered_firetext_callback(): + assert firetext_callback("1234", "07811111111") == "mobile=07811111111&status=0&time=2016-03-10 14:17:00&reference=1234" # noqa + + +def test_failure_firetext_callback(): + assert firetext_callback("1234", "07822222222") == "mobile=07822222222&status=1&time=2016-03-10 14:17:00&reference=1234" # noqa + + +def test_delivered_ses_callback(): + assert ses_notification_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"ref\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa + + +def test_ses_hard_bounce_callback(): + assert ses_hard_bounce_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"ref\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa + + +def ses_soft_bounce_callback(): + assert ses_soft_bounce_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"ref\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index d6aa09d9a..7a4e24180 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -17,6 +17,10 @@ from app.celery.tasks import ( delete_successful_notifications, provider_to_use ) +from app.celery.research_mode_tasks import ( + send_email_response, + send_sms_response +) from app import (aws_ses_client, encryption, DATETIME_FORMAT, mmg_client, statsd_client) from app.clients.email.aws_ses import AwsSesClientException from app.clients.sms.mmg import MMGClientException @@ -993,6 +997,86 @@ def test_process_email_job_should_use_reply_to_email_if_present(sample_email_job ) +def test_should_call_send_sms_response_task_if_research_mode(notify_db, sample_service, sample_template, mocker): + notification = _notification_json( + sample_template, + to="+447234123123" + ) + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.mmg_client.send_sms') + mocker.patch('app.mmg_client.get_name', return_value="mmg") + mocker.patch('app.celery.research_mode_tasks.send_sms_response.apply_async') + + sample_service.research_mode = True + notify_db.session.add(sample_service) + notify_db.session.commit() + + notification_id = uuid.uuid4() + now = datetime.utcnow() + send_sms( + sample_service.id, + notification_id, + "encrypted-in-reality", + now.strftime(DATETIME_FORMAT) + ) + assert not mmg_client.send_sms.called + send_sms_response.apply_async.assert_called_once_with(('mmg', str(notification_id), "+447234123123")) + + persisted_notification = notifications_dao.get_notification(sample_service.id, notification_id) + assert persisted_notification.id == notification_id + assert persisted_notification.to == '+447234123123' + assert persisted_notification.template_id == sample_template.id + assert persisted_notification.status == 'sending' + assert persisted_notification.sent_at > now + assert persisted_notification.created_at == now + assert persisted_notification.sent_by == 'mmg' + + +def test_should_call_send_email_response_task_if_research_mode( + notify_db, + sample_service, + sample_email_template, + mocker): + notification = _notification_json( + sample_email_template, + to="john@smith.com" + ) + + reference = uuid.uuid4() + + mocker.patch('app.uuid.uuid4', return_value=reference) + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.get_name', return_value="ses") + mocker.patch('app.celery.research_mode_tasks.send_email_response.apply_async') + + sample_service.research_mode = True + notify_db.session.add(sample_service) + notify_db.session.commit() + + notification_id = uuid.uuid4() + now = datetime.utcnow() + send_email( + sample_service.id, + notification_id, + "myservice@notify.com", + "encrypted-in-reality", + now.strftime(DATETIME_FORMAT) + ) + assert not aws_ses_client.send_email.called + send_email_response.apply_async.assert_called_once_with(('ses', str(reference), 'john@smith.com')) + + persisted_notification = notifications_dao.get_notification(sample_service.id, notification_id) + assert persisted_notification.id == notification_id + assert persisted_notification.to == 'john@smith.com' + assert persisted_notification.template_id == sample_email_template.id + assert persisted_notification.status == 'sending' + assert persisted_notification.sent_at > now + assert persisted_notification.created_at == now + assert persisted_notification.sent_by == 'ses' + assert persisted_notification.reference == str(reference) + + def _notification_json(template, to, personalisation=None, job_id=None, row_number=None): notification = { "template": template.id, diff --git a/tests/app/conftest.py b/tests/app/conftest.py index cdb4cb0ec..fc3d31dea 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -1,3 +1,4 @@ +import requests_mock import pytest from datetime import (datetime, date) from app import db @@ -23,6 +24,12 @@ from app.dao.invited_user_dao import save_invited_user import uuid +@pytest.yield_fixture +def rmock(): + with requests_mock.mock() as rmock: + yield rmock + + @pytest.fixture(scope='function') def service_factory(notify_db, notify_db_session): class ServiceFactory(object): From 290f416485ff7903575d590e481c93c0b21ff683 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 1 Jun 2016 16:57:57 +0100 Subject: [PATCH 3/9] Various tidy ups and changes - use new queue - remove some new lines in AWS response data - migration script to populate new fields --- app/celery/research_mode_tasks.py | 8 +++---- app/celery/tasks.py | 8 +++++-- config.py | 3 ++- .../0024_add_research_mode_defaults.py | 22 +++++++++++++++++++ tests/app/celery/test_research_mode_tasks.py | 10 ++++----- tests/app/celery/test_tasks.py | 8 +++++-- 6 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 migrations/versions/0024_add_research_mode_defaults.py diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index 0e76d5158..907716706 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -18,7 +18,7 @@ def send_sms_response(provider, reference, to): if provider == "mmg": body = mmg_callback(reference, to) headers = {"Content-type": "application/json"} - if provider == "firetext": + else: headers = {"Content-type": "text/plain"} body = firetext_callback(reference, to) make_request('sms', provider, body, headers) @@ -96,12 +96,12 @@ def firetext_callback(notification_id, to): def ses_notification_callback(reference): - return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa + return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa def ses_hard_bounce_callback(reference): - return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa + return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa def ses_soft_bounce_callback(reference): - return '{\n "Type" : "Notification",\n "MessageId" : "%s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % (reference, reference) # noqa + return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa diff --git a/app/celery/tasks.py b/app/celery/tasks.py index e68169fc5..ee33ca1c8 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -263,7 +263,9 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): try: if service.research_mode: - send_sms_response.apply_async((provider.get_name(), str(notification_id), notification['to']), queue='sms') + send_sms_response.apply_async( + (provider.get_name(), str(notification_id), notification['to']), queue='research-mode' + ) else: provider.send_sms( to=validate_and_format_phone_number(notification['to']), @@ -338,7 +340,9 @@ def send_email(service_id, notification_id, from_address, encrypted_notification if service.research_mode: reference = create_uuid() - send_email_response.apply_async((provider.get_name(), str(reference), notification['to']), queue='email') + send_email_response.apply_async( + (provider.get_name(), str(reference), notification['to']), queue='research-mode' + ) else: reference = provider.send_email( from_address, diff --git a/config.py b/config.py index f3cc20aca..715bbfe3e 100644 --- a/config.py +++ b/config.py @@ -75,7 +75,8 @@ class Config(object): Queue('bulk-sms', Exchange('default'), routing_key='bulk-sms'), Queue('bulk-email', Exchange('default'), routing_key='bulk-email'), Queue('email-invited-user', Exchange('default'), routing_key='email-invited-user'), - Queue('email-registration-verification', Exchange('default'), routing_key='email-registration-verification') + Queue('email-registration-verification', Exchange('default'), routing_key='email-registration-verification'), + Queue('research-mode', Exchange('default'), routing_key='research-mode') ] TWILIO_ACCOUNT_SID = os.getenv('TWILIO_ACCOUNT_SID') TWILIO_AUTH_TOKEN = os.getenv('TWILIO_AUTH_TOKEN') diff --git a/migrations/versions/0024_add_research_mode_defaults.py b/migrations/versions/0024_add_research_mode_defaults.py new file mode 100644 index 000000000..83ea22943 --- /dev/null +++ b/migrations/versions/0024_add_research_mode_defaults.py @@ -0,0 +1,22 @@ +"""empty message + +Revision ID: 0024_add_research_mode_defaults +Revises: 0023_add_research_mode +Create Date: 2016-05-31 11:11:45.979594 + +""" + +# revision identifiers, used by Alembic. +revision = '0024_add_research_mode_defaults' +down_revision = '0023_add_research_mode' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.execute('update services set research_mode = false') + op.execute('update services_history set research_mode = false') + +def downgrade(): + pass \ No newline at end of file diff --git a/tests/app/celery/test_research_mode_tasks.py b/tests/app/celery/test_research_mode_tasks.py index c82f8b955..8c99e98c5 100644 --- a/tests/app/celery/test_research_mode_tasks.py +++ b/tests/app/celery/test_research_mode_tasks.py @@ -27,7 +27,7 @@ def test_make_firetext_callback(notify_api, rmock): rmock.request( "POST", endpoint, - data="some data", + json="some data", status_code=200) send_sms_response("firetext", "1234", "07811111111") @@ -49,7 +49,7 @@ def test_make_ses_callback(notify_api, rmock): def test_delivered_mmg_callback(): data = json.loads(mmg_callback("1234", "07811111111")) assert data['MSISDN'] == "07811111111" - assert data['status'] == "0" + assert data['status'] == "3" assert data['reference'] == "mmg_reference" assert data['CID'] == "1234" @@ -79,12 +79,12 @@ def test_failure_firetext_callback(): def test_delivered_ses_callback(): - assert ses_notification_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"ref\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa + assert ses_notification_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"my-reference\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa def test_ses_hard_bounce_callback(): - assert ses_hard_bounce_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"ref\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa + assert ses_hard_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"my-reference\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa def ses_soft_bounce_callback(): - assert ses_soft_bounce_callback("my-reference") == '{\n "Type" : "Notification",\n "MessageId" : "my-reference",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"ref\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' # noqa + assert ses_soft_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 7a4e24180..b43d3d96a 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1020,7 +1020,9 @@ def test_should_call_send_sms_response_task_if_research_mode(notify_db, sample_s now.strftime(DATETIME_FORMAT) ) assert not mmg_client.send_sms.called - send_sms_response.apply_async.assert_called_once_with(('mmg', str(notification_id), "+447234123123")) + send_sms_response.apply_async.assert_called_once_with( + ('mmg', str(notification_id), "+447234123123"), queue='research-mode' + ) persisted_notification = notifications_dao.get_notification(sample_service.id, notification_id) assert persisted_notification.id == notification_id @@ -1064,7 +1066,9 @@ def test_should_call_send_email_response_task_if_research_mode( now.strftime(DATETIME_FORMAT) ) assert not aws_ses_client.send_email.called - send_email_response.apply_async.assert_called_once_with(('ses', str(reference), 'john@smith.com')) + send_email_response.apply_async.assert_called_once_with( + ('ses', str(reference), 'john@smith.com'), queue="research-mode" + ) persisted_notification = notifications_dao.get_notification(sample_service.id, notification_id) assert persisted_notification.id == notification_id From 6ca2e588a9502790f2c7b0da13e0d5cedb40709e Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 08:59:30 +0100 Subject: [PATCH 4/9] Re-aligned the dao update that moved by mistake --- app/celery/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index c063214fe..1cd97dd98 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -274,8 +274,8 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): ) current_app.logger.exception(e) notification_db_object.status = 'technical-failure' + dao_update_notification(notification_db_object) - dao_update_notification(notification_db_object) current_app.logger.info( "SMS {} created at {} sent at {}".format(notification_id, created_at, sent_at) ) From b08f906662083cb4ca703c1553dd9842cbaeaa4f Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 09:28:21 +0100 Subject: [PATCH 5/9] Added 2 tests to ensure provider stats not updated when in research nmode --- tests/app/celery/test_tasks.py | 90 +++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 7cb5c2d34..8315ad362 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -29,6 +29,7 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm.exc import NoResultFound from app.celery.tasks import s3 from app.celery import tasks +from app.dao.provider_statistics_dao import get_provider_statistics from tests.app import load_example_csv from datetime import datetime, timedelta from freezegun import freeze_time @@ -331,7 +332,7 @@ def test_should_process_all_sms_job(sample_job, assert encryption.encrypt.call_args[0][0]['to'] == '+441234123120' assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id) assert encryption.encrypt.call_args[0][0][ - 'template_version'] == sample_job_with_placeholdered_template.template.version + 'template_version'] == sample_job_with_placeholdered_template.template.version assert encryption.encrypt.call_args[0][0]['personalisation'] == {'name': 'chris'} tasks.send_sms.apply_async.call_count == 10 job = jobs_dao.dao_get_job_by_id(sample_job_with_placeholdered_template.id) @@ -1084,6 +1085,93 @@ def test_should_call_send_email_response_task_if_research_mode( assert persisted_notification.reference == str(reference) +def test_should_call_send_not_update_provider_email_stats_if_research_mode( + notify_db, + sample_service, + sample_email_template, + ses_provider, + mocker): + notification = _notification_json( + sample_email_template, + to="john@smith.com" + ) + + reference = uuid.uuid4() + + mocker.patch('app.uuid.uuid4', return_value=reference) + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.get_name', return_value="ses") + mocker.patch('app.celery.research_mode_tasks.send_email_response.apply_async') + + sample_service.research_mode = True + notify_db.session.add(sample_service) + notify_db.session.commit() + + assert not get_provider_statistics( + sample_email_template.service, + providers=[ses_provider.identifier]).first() + + notification_id = uuid.uuid4() + now = datetime.utcnow() + send_email( + sample_service.id, + notification_id, + "myservice@notify.com", + "encrypted-in-reality", + now.strftime(DATETIME_FORMAT) + ) + assert not aws_ses_client.send_email.called + send_email_response.apply_async.assert_called_once_with( + ('ses', str(reference), 'john@smith.com'), queue="research-mode" + ) + + assert not get_provider_statistics( + sample_email_template.service, + providers=[ses_provider.identifier]).first() + + +def test_should_call_send_sms_response_task_if_research_mode( + notify_db, + sample_service, + sample_template, + mmg_provider, + mocker): + notification = _notification_json( + sample_template, + to="+447234123123" + ) + mocker.patch('app.encryption.decrypt', return_value=notification) + mocker.patch('app.mmg_client.send_sms') + mocker.patch('app.mmg_client.get_name', return_value="mmg") + mocker.patch('app.celery.research_mode_tasks.send_sms_response.apply_async') + + sample_service.research_mode = True + notify_db.session.add(sample_service) + notify_db.session.commit() + + assert not get_provider_statistics( + sample_template.service, + providers=[mmg_provider.identifier]).first() + + notification_id = uuid.uuid4() + now = datetime.utcnow() + send_sms( + sample_service.id, + notification_id, + "encrypted-in-reality", + now.strftime(DATETIME_FORMAT) + ) + assert not mmg_client.send_sms.called + send_sms_response.apply_async.assert_called_once_with( + ('mmg', str(notification_id), "+447234123123"), queue='research-mode' + ) + + assert not get_provider_statistics( + sample_template.service, + providers=[mmg_provider.identifier]).first() + + def _notification_json(template, to, personalisation=None, job_id=None, row_number=None): notification = { "template": template.id, From 099c17192d1d8e65284266aa5335eeafd05010aa Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 09:30:01 +0100 Subject: [PATCH 6/9] Merged provider stats only on success branch - ensures that don't raise stats in research mode --- app/celery/tasks.py | 22 +++++++++++----------- tests/app/celery/test_tasks.py | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 761e01543..b20fe59c2 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -268,11 +268,11 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): reference=str(notification_id) ) - update_notification_after_sent_to_provider( - notification_id, - 'sms', - provider.get_name() - ) + update_notification_after_sent_to_provider( + notification_id, + 'sms', + provider.get_name() + ) except SmsClientException as e: current_app.logger.error( @@ -349,12 +349,12 @@ def send_email(service_id, notification_id, from_address, encrypted_notification reply_to_addresses=reply_to_addresses, ) - update_notification_after_sent_to_provider( - notification_id, - 'email', - provider.get_name(), - reference=reference - ) + update_notification_after_sent_to_provider( + notification_id, + 'email', + provider.get_name(), + reference=reference + ) except EmailClientException as e: current_app.logger.exception(e) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 8315ad362..e73c0d391 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -331,8 +331,7 @@ def test_should_process_all_sms_job(sample_job, ) assert encryption.encrypt.call_args[0][0]['to'] == '+441234123120' assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id) - assert encryption.encrypt.call_args[0][0][ - 'template_version'] == sample_job_with_placeholdered_template.template.version + assert encryption.encrypt.call_args[0][0]['template_version'] == sample_job_with_placeholdered_template.template.version # noqa assert encryption.encrypt.call_args[0][0]['personalisation'] == {'name': 'chris'} tasks.send_sms.apply_async.call_count == 10 job = jobs_dao.dao_get_job_by_id(sample_job_with_placeholdered_template.id) From 754ccbe9af3f701f963748edf005e7448c1d0196 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 09:52:47 +0100 Subject: [PATCH 7/9] Removed update reference from updating the provider stats - single focus method - allows not to pollute DAO with research mode --- app/celery/tasks.py | 12 +++++++----- app/dao/notifications_dao.py | 12 ++++++------ tests/app/celery/test_tasks.py | 8 ++++---- tests/app/dao/test_notification_dao.py | 10 +++++----- ...st_notifications_dao_provider_statistics.py | 18 +++++++++--------- 5 files changed, 31 insertions(+), 29 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index b20fe59c2..80925b5af 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -37,7 +37,7 @@ from app.dao.notifications_dao import ( dao_update_notification, delete_notifications_created_more_than_a_week_ago, dao_get_notification_statistics_for_service_and_day, - update_notification_after_sent_to_provider + update_provider_stats ) from app.dao.jobs_dao import ( @@ -268,7 +268,7 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): reference=str(notification_id) ) - update_notification_after_sent_to_provider( + update_provider_stats( notification_id, 'sms', provider.get_name() @@ -349,13 +349,15 @@ def send_email(service_id, notification_id, from_address, encrypted_notification reply_to_addresses=reply_to_addresses, ) - update_notification_after_sent_to_provider( + update_provider_stats( notification_id, 'email', - provider.get_name(), - reference=reference + provider.get_name() ) + notification_db_object.reference = reference + dao_update_notification(notification_db_object) + except EmailClientException as e: current_app.logger.exception(e) notification_db_object.status = 'technical-failure' diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index c5db52329..ac9d597d9 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -278,9 +278,13 @@ def dao_update_notification(notification): @transactional -def update_notification_after_sent_to_provider(id_, notification_type, provider_name, reference=None): - provider = ProviderDetails.query.filter_by(identifier=provider_name).one() +def update_provider_stats( + id_, + notification_type, + provider_name): + notification = Notification.query.filter(Notification.id == id_).one() + provider = ProviderDetails.query.filter_by(identifier=provider_name).one() def unit_count(): if notification_type == TEMPLATE_TYPE_EMAIL: @@ -304,10 +308,6 @@ def update_notification_after_sent_to_provider(id_, notification_type, provider_ db.session.add(provider_stats) - if reference: - notification.reference = reference - db.session.add(notification) - def get_notification_for_job(service_id, job_id, notification_id): return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one() diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index e73c0d391..e0b8c7166 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -518,7 +518,7 @@ def test_should_send_email_if_restricted_service_and_valid_email(notify_db, noti notification = _notification_json(template, "test@restricted.com") mocker.patch('app.encryption.decrypt', return_value=notification) - mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.send_email', return_value="1234") notification_id = uuid.uuid4() now = datetime.utcnow() @@ -667,7 +667,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh def test_send_email_should_use_template_version_from_job_not_latest(sample_email_template, mocker): notification = _notification_json(sample_email_template, 'my_email@my_email.com') mocker.patch('app.encryption.decrypt', return_value=notification) - mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.send_email', return_value="1234") mocker.patch('app.aws_ses_client.get_name', return_value='ses') version_on_notification = sample_email_template.version # Change the template @@ -710,7 +710,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi notification = _notification_json(sample_email_template_with_placeholders, "my_email@my_email.com", {"name": "Jo"}) mocker.patch('app.encryption.decrypt', return_value=notification) - mocker.patch('app.aws_ses_client.send_email') + mocker.patch('app.aws_ses_client.send_email', return_value="1234") mocker.patch('app.aws_ses_client.get_name', return_value='ses') notification_id = uuid.uuid4() @@ -1046,7 +1046,6 @@ def test_should_call_send_email_response_task_if_research_mode( sample_email_template, to="john@smith.com" ) - reference = uuid.uuid4() mocker.patch('app.uuid.uuid4', return_value=reference) @@ -1060,6 +1059,7 @@ def test_should_call_send_email_response_task_if_research_mode( notify_db.session.commit() notification_id = uuid.uuid4() + now = datetime.utcnow() send_email( sample_service.id, diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index a26dbbaa6..94991d63b 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -25,7 +25,7 @@ from app.dao.notifications_dao import ( delete_notifications_created_more_than_a_week_ago, dao_get_notification_statistics_for_service_and_day, update_notification_status_by_id, - update_notification_after_sent_to_provider, + update_provider_stats, update_notification_status_by_reference, dao_get_template_statistics_for_service, get_notifications_for_service @@ -38,7 +38,7 @@ from tests.app.conftest import (sample_notification) def test_should_by_able_to_update_reference_by_notification_id(sample_notification, mmg_provider): assert not Notification.query.get(sample_notification.id).reference - update_notification_after_sent_to_provider( + update_provider_stats( sample_notification.id, 'sms', mmg_provider.identifier, @@ -56,7 +56,7 @@ def test_should_by_able_to_update_status_by_reference(sample_email_template, ses ses_provider.identifier) assert Notification.query.get(notification.id).status == "sending" - update_notification_after_sent_to_provider( + update_provider_stats( notification.id, 'email', ses_provider.identifier, @@ -147,7 +147,7 @@ def test_should_not_update_status_one_notification_status_is_delivered(sample_em ses_provider.identifier) assert Notification.query.get(notification.id).status == "sending" - update_notification_after_sent_to_provider( + update_provider_stats( notification.id, 'email', ses_provider.identifier, @@ -174,7 +174,7 @@ def test_should_be_able_to_record_statistics_failure_for_email(sample_email_temp notification = Notification(**data) dao_create_notification(notification, sample_email_template.template_type, ses_provider.identifier) - update_notification_after_sent_to_provider( + update_provider_stats( notification.id, 'email', ses_provider.identifier, diff --git a/tests/app/dao/test_notifications_dao_provider_statistics.py b/tests/app/dao/test_notifications_dao_provider_statistics.py index 24d2fc6dd..5a096c114 100644 --- a/tests/app/dao/test_notifications_dao_provider_statistics.py +++ b/tests/app/dao/test_notifications_dao_provider_statistics.py @@ -1,6 +1,6 @@ from datetime import (date, timedelta) from app.models import ProviderStatistics -from app.dao.notifications_dao import update_notification_after_sent_to_provider +from app.dao.notifications_dao import update_provider_stats from app.dao.provider_statistics_dao import ( get_provider_statistics, get_fragment_count) from tests.app.conftest import sample_notification as create_sample_notification @@ -14,7 +14,7 @@ def test_should_update_provider_statistics_sms(notify_db, notify_db, notify_db_session, template=sample_template) - update_notification_after_sent_to_provider(n1.id, 'sms', mmg_provider.identifier) + update_provider_stats(n1.id, 'sms', mmg_provider.identifier) provider_stats = get_provider_statistics( sample_template.service, providers=[mmg_provider.identifier]).one() @@ -29,7 +29,7 @@ def test_should_update_provider_statistics_email(notify_db, notify_db, notify_db_session, template=sample_email_template) - update_notification_after_sent_to_provider(n1.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n1.id, 'email', ses_provider.identifier, reference="reference") provider_stats = get_provider_statistics( sample_email_template.service, providers=[ses_provider.identifier]).one() @@ -46,21 +46,21 @@ def test_should_update_provider_statistics_sms_multi(notify_db, template=sample_template, provider_name=mmg_provider.identifier, content_char_count=160) - update_notification_after_sent_to_provider(n1.id, 'sms', mmg_provider.identifier) + update_provider_stats(n1.id, 'sms', mmg_provider.identifier) n2 = create_sample_notification( notify_db, notify_db_session, template=sample_template, provider_name=mmg_provider.identifier, content_char_count=161) - update_notification_after_sent_to_provider(n2.id, 'sms', mmg_provider.identifier) + update_provider_stats(n2.id, 'sms', mmg_provider.identifier) n3 = create_sample_notification( notify_db, notify_db_session, template=sample_template, provider_name=mmg_provider.identifier, content_char_count=307) - update_notification_after_sent_to_provider(n3.id, 'sms', mmg_provider.identifier) + update_provider_stats(n3.id, 'sms', mmg_provider.identifier) provider_stats = get_provider_statistics( sample_template.service, providers=[mmg_provider.identifier]).one() @@ -76,19 +76,19 @@ def test_should_update_provider_statistics_email_multi(notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_notification_after_sent_to_provider(n1.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n1.id, 'email', ses_provider.identifier, reference="reference") n2 = create_sample_notification( notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_notification_after_sent_to_provider(n2.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n2.id, 'email', ses_provider.identifier, reference="reference") n3 = create_sample_notification( notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_notification_after_sent_to_provider(n3.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n3.id, 'email', ses_provider.identifier, reference="reference") provider_stats = get_provider_statistics( sample_email_template.service, providers=[ses_provider.identifier]).one() From c6c534365e591efe004829e701079ea3f394f1df Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 10:00:27 +0100 Subject: [PATCH 8/9] Fixing tests as stats update now don't update reference --- tests/app/dao/test_notification_dao.py | 27 +++++++------------ ...t_notifications_dao_provider_statistics.py | 8 +++--- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 94991d63b..e25c356c3 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -36,15 +36,6 @@ from notifications_utils.template import get_sms_fragment_count from tests.app.conftest import (sample_notification) -def test_should_by_able_to_update_reference_by_notification_id(sample_notification, mmg_provider): - assert not Notification.query.get(sample_notification.id).reference - update_provider_stats( - sample_notification.id, - 'sms', - mmg_provider.identifier, - reference='reference') - assert Notification.query.get(sample_notification.id).reference == 'reference' - def test_should_by_able_to_update_status_by_reference(sample_email_template, ses_provider): data = _notification_json(sample_email_template) @@ -56,11 +47,9 @@ def test_should_by_able_to_update_status_by_reference(sample_email_template, ses ses_provider.identifier) assert Notification.query.get(notification.id).status == "sending" - update_provider_stats( - notification.id, - 'email', - ses_provider.identifier, - reference="reference") + notification.reference = 'reference' + dao_update_notification(notification) + update_notification_status_by_reference('reference', 'delivered', 'delivered') assert Notification.query.get(notification.id).status == 'delivered' _assert_notification_stats(notification.service_id, emails_delivered=1, emails_requested=1, emails_failed=0) @@ -150,8 +139,9 @@ def test_should_not_update_status_one_notification_status_is_delivered(sample_em update_provider_stats( notification.id, 'email', - ses_provider.identifier, - reference='reference') + ses_provider.identifier) + notification.reference = 'reference' + dao_update_notification(notification) update_notification_status_by_reference('reference', 'delivered', 'delivered') assert Notification.query.get(notification.id).status == 'delivered' @@ -177,8 +167,9 @@ def test_should_be_able_to_record_statistics_failure_for_email(sample_email_temp update_provider_stats( notification.id, 'email', - ses_provider.identifier, - reference='reference') + ses_provider.identifier) + notification.reference = 'reference' + dao_update_notification(notification) count = update_notification_status_by_reference('reference', 'failed', 'failure') assert count == 1 assert Notification.query.get(notification.id).status == 'failed' diff --git a/tests/app/dao/test_notifications_dao_provider_statistics.py b/tests/app/dao/test_notifications_dao_provider_statistics.py index 5a096c114..7c7abcfcb 100644 --- a/tests/app/dao/test_notifications_dao_provider_statistics.py +++ b/tests/app/dao/test_notifications_dao_provider_statistics.py @@ -29,7 +29,7 @@ def test_should_update_provider_statistics_email(notify_db, notify_db, notify_db_session, template=sample_email_template) - update_provider_stats(n1.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n1.id, 'email', ses_provider.identifier) provider_stats = get_provider_statistics( sample_email_template.service, providers=[ses_provider.identifier]).one() @@ -76,19 +76,19 @@ def test_should_update_provider_statistics_email_multi(notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_provider_stats(n1.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n1.id, 'email', ses_provider.identifier) n2 = create_sample_notification( notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_provider_stats(n2.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n2.id, 'email', ses_provider.identifier) n3 = create_sample_notification( notify_db, notify_db_session, template=sample_email_template, provider_name=ses_provider.identifier) - update_provider_stats(n3.id, 'email', ses_provider.identifier, reference="reference") + update_provider_stats(n3.id, 'email', ses_provider.identifier) provider_stats = get_provider_statistics( sample_email_template.service, providers=[ses_provider.identifier]).one() From 5e1613e461f38a6c9e4c2b900b1d975d51d7d1a1 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Thu, 2 Jun 2016 10:35:54 +0100 Subject: [PATCH 9/9] pep8 too many lines --- tests/app/dao/test_notification_dao.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index e25c356c3..e4a7d9694 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -36,7 +36,6 @@ from notifications_utils.template import get_sms_fragment_count from tests.app.conftest import (sample_notification) - def test_should_by_able_to_update_status_by_reference(sample_email_template, ses_provider): data = _notification_json(sample_email_template) @@ -330,7 +329,6 @@ def test_save_notification_creates_sms_and_template_stats(sample_template, sampl def test_save_notification_and_create_email_and_template_stats(sample_email_template, sample_job, ses_provider): - assert Notification.query.count() == 0 assert NotificationStatistics.query.count() == 0 assert TemplateStatistics.query.count() == 0 @@ -619,7 +617,6 @@ def test_update_notification(sample_notification, sample_template): @freeze_time("2016-01-10 12:00:00.000000") def test_should_delete_notifications_after_seven_days(notify_db, notify_db_session): - assert len(Notification.query.all()) == 0 # create one notification a day between 1st and 9th from 11:00 to 19:00 @@ -769,7 +766,6 @@ def test_successful_notification_inserts_followed_by_failure_does_not_increment_ def test_get_template_stats_for_service_returns_stats_in_reverse_date_order(sample_template, sample_job, mmg_provider): - template_stats = dao_get_template_statistics_for_service(sample_template.service.id) assert len(template_stats) == 0 data = _notification_json(sample_template, job_id=sample_job.id) @@ -796,7 +792,6 @@ def test_get_template_stats_for_service_returns_stats_in_reverse_date_order(samp @freeze_time('2016-04-09') def test_get_template_stats_for_service_returns_stats_can_limit_number_of_days_returned(sample_template): - template_stats = dao_get_template_statistics_for_service(sample_template.service.id) assert len(template_stats) == 0 @@ -820,7 +815,6 @@ def test_get_template_stats_for_service_returns_stats_can_limit_number_of_days_r @freeze_time('2016-04-09') def test_get_template_stats_for_service_returns_stats_returns_all_stats_if_no_limit(sample_template): - template_stats = dao_get_template_statistics_for_service(sample_template.service.id) assert len(template_stats) == 0 @@ -841,7 +835,6 @@ def test_get_template_stats_for_service_returns_stats_returns_all_stats_if_no_li @freeze_time('2016-04-30') def test_get_template_stats_for_service_returns_no_result_if_no_usage_within_limit_days(sample_template): - template_stats = dao_get_template_statistics_for_service(sample_template.service.id) assert len(template_stats) == 0 @@ -870,7 +863,6 @@ def test_get_template_stats_for_service_with_limit_if_no_records_returns_empty_l @freeze_time("2016-01-10") def test_should_limit_notifications_return_by_day_limit_plus_one(notify_db, notify_db_session, sample_service): - assert len(Notification.query.all()) == 0 # create one notification a day between 1st and 9th