From 439722990ed0b4145cb83d86559accd62b589d20 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 28 Feb 2023 14:49:53 -0500 Subject: [PATCH 1/7] Bind to sns service --- deploy-config/demo.yml | 1 + deploy-config/production.yml | 1 + deploy-config/sandbox.yml | 1 + deploy-config/staging.yml | 1 + manifest.yml | 3 ++- 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy-config/demo.yml b/deploy-config/demo.yml index 133630da0..aede2dcc2 100644 --- a/deploy-config/demo.yml +++ b/deploy-config/demo.yml @@ -6,3 +6,4 @@ worker_memory: 512M scheduler_memory: 256M public_api_route: notify-api-demo.app.cloud.gov admin_base_url: https://notify-demo.app.cloud.gov +default_toll_free_number: "+18337581259" diff --git a/deploy-config/production.yml b/deploy-config/production.yml index c1bbf4832..911b4b97c 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -6,3 +6,4 @@ worker_memory: 512M scheduler_memory: 256M public_api_route: notify-api.app.cloud.gov admin_base_url: https://notify.app.cloud.gov +default_toll_free_number: "" diff --git a/deploy-config/sandbox.yml b/deploy-config/sandbox.yml index aafa9b491..83ec98af4 100644 --- a/deploy-config/sandbox.yml +++ b/deploy-config/sandbox.yml @@ -9,3 +9,4 @@ admin_base_url: https://notify-sandbox.app.cloud.gov ADMIN_CLIENT_SECRET: sandbox-notify-secret-key DANGEROUS_SALT: sandbox-notify-salt SECRET_KEY: sandbox-notify-secret-key +default_toll_free_number: "" diff --git a/deploy-config/staging.yml b/deploy-config/staging.yml index 96343f804..f6a2de501 100644 --- a/deploy-config/staging.yml +++ b/deploy-config/staging.yml @@ -6,3 +6,4 @@ worker_memory: 512M scheduler_memory: 256M public_api_route: notify-api-staging.app.cloud.gov admin_base_url: https://notify-staging.app.cloud.gov +default_toll_free_number: "+18556438890" diff --git a/manifest.yml b/manifest.yml index 71c91a877..55a1252d9 100644 --- a/manifest.yml +++ b/manifest.yml @@ -16,6 +16,7 @@ applications: - name: notify-api-ses-((env)) parameters: notification_webhook: "https://((public_api_route))/notifications/email/ses" + - notify-api-sns-((env)) processes: - type: web @@ -51,7 +52,7 @@ applications: AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 - AWS_US_TOLL_FREE_NUMBER: +18446120782 + AWS_US_TOLL_FREE_NUMBER: ((default_toll_free_number)) REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" NEW_RELIC_CA_BUNDLE_PATH: "/etc/ssl/certs/ca-certificates.crt" From 28f8649444d7f95b74dcc42ea1578916e24ce5b1 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 28 Feb 2023 16:50:00 -0500 Subject: [PATCH 2/7] Use sns credentials from VCAP_SERVICES --- .github/workflows/checks.yml | 1 - .github/workflows/daily_checks.yml | 1 - app/clients/sms/aws_sns.py | 33 +++++++++++------------------- app/cloudfoundry_config.py | 24 ++++++++++++++++++++++ app/config.py | 1 - manifest.yml | 2 -- sample.env | 1 - tests/app/clients/test_aws_sns.py | 5 ++++- 8 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 194d9bf14..f4c8c38df 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,7 +14,6 @@ env: WERKZEUG_DEBUG_PIN: off REDIS_ENABLED: 0 AWS_REGION: us-west-2 - AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 jobs: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index fa4f6773a..a1c452e8e 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -19,7 +19,6 @@ env: NOTIFY_EMAIL_DOMAIN: dispostable.com REDIS_ENABLED: 0 AWS_REGION: us-west-2 - AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 jobs: diff --git a/app/clients/sms/aws_sns.py b/app/clients/sms/aws_sns.py index 4637bdfe4..36ebbddfe 100644 --- a/app/clients/sms/aws_sns.py +++ b/app/clients/sms/aws_sns.py @@ -1,11 +1,11 @@ -import re from time import monotonic -import boto3 import botocore import phonenumbers +from boto3 import client from app.clients.sms import SmsClient +from app.cloudfoundry_config import cloud_config class AwsSnsClient(SmsClient): @@ -14,19 +14,22 @@ class AwsSnsClient(SmsClient): """ def init_app(self, current_app, statsd_client, *args, **kwargs): - self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"]) - self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"]) + self._client = client( + "sns", + region_name=cloud_config.sns_region, + aws_access_key_id=cloud_config.sns_access_key, + aws_secret_access_key=cloud_config.sns_secret_key + ) super(SmsClient, self).__init__(*args, **kwargs) self.current_app = current_app self.statsd_client = statsd_client - self.long_code_regex = re.compile(r"^\+1\d{10}$") @property def name(self): return 'sns' def get_name(self): - return 'sns' + return self.name def send_sms(self, to, content, reference, sender=None, international=False): matched = False @@ -35,7 +38,6 @@ class AwsSnsClient(SmsClient): matched = True to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164) - client = self._client # See documentation # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk attributes = { @@ -45,20 +47,12 @@ class AwsSnsClient(SmsClient): } } - # If sending with a long code number, we need to use another AWS region - # and specify the phone number we want to use as the origination number - send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender) - if send_with_dedicated_phone_number: - client = self._long_codes_client + if sender: attributes["AWS.MM.SMS.OriginationNumber"] = { "DataType": "String", "StringValue": sender, } - - # If the number is US based, we must use a US Toll Free number to send the message - country = phonenumbers.region_code_for_number(match.number) - if country == "US": - client = self._long_codes_client + else: attributes["AWS.MM.SMS.OriginationNumber"] = { "DataType": "String", "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"], @@ -66,7 +60,7 @@ class AwsSnsClient(SmsClient): try: start_time = monotonic() - response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes) + response = self._client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes) except botocore.exceptions.ClientError as e: self.statsd_client.incr("clients.sns.error") raise str(e) @@ -84,6 +78,3 @@ class AwsSnsClient(SmsClient): self.statsd_client.incr("clients.sns.error") self.current_app.logger.error("No valid numbers found in {}".format(to)) raise ValueError("No valid numbers found for SMS delivery") - - def _send_with_dedicated_phone_number(self, sender): - return sender and re.match(self.long_code_regex, sender) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index a23ee260c..a6c3ad075 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -59,6 +59,27 @@ class CloudfoundryConfig: except KeyError: return getenv('AWS_SECRET_ACCESS_KEY') + @property + def sns_access_key(self): + try: + return self._sns_credentials('aws_access_key_id') + except KeyError: + return getenv('AWS_ACCESS_KEY_ID') + + @property + def sns_secret_key(self): + try: + return self._sns_credentials('aws_secret_access_key') + except KeyError: + return getenv('AWS_SECRET_ACCESS_KEY') + + @property + def sns_region(self): + try: + return self._sns_credentials('region') + except KeyError: + return getenv('AWS_REGION') + @property def sns_topic_arns(self): try: @@ -73,5 +94,8 @@ class CloudfoundryConfig: def _ses_credentials(self, key): return self.parsed_services['datagov-smtp'][0]['credentials'][key] + def _sns_credentials(self, key): + return self.parsed_services['ttsnotify-sms'][0]['credentials'][key] + cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index d0201086f..cca2b5e60 100644 --- a/app/config.py +++ b/app/config.py @@ -104,7 +104,6 @@ class Config(object): # AWS Settings AWS_REGION = getenv('AWS_REGION') - AWS_PINPOINT_REGION = getenv("AWS_PINPOINT_REGION") AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER") # Whether to ignore POSTs from SNS for replies to SMS we sent RECEIVE_INBOUND_SMS = False diff --git a/manifest.yml b/manifest.yml index 55a1252d9..7db86dd06 100644 --- a/manifest.yml +++ b/manifest.yml @@ -50,8 +50,6 @@ applications: SECRET_KEY: ((SECRET_KEY)) AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) - AWS_REGION: us-west-2 - AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: ((default_toll_free_number)) REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" diff --git a/sample.env b/sample.env index ced1198f7..1fd0c4998 100644 --- a/sample.env +++ b/sample.env @@ -19,7 +19,6 @@ AWS_ACCESS_KEY_ID="don't write secrets to the sample file" AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" AWS_REGION=us-west-2 -AWS_PINPOINT_REGION=us-west-2 AWS_US_TOLL_FREE_NUMBER=+18446120782 ############################################################# diff --git a/tests/app/clients/test_aws_sns.py b/tests/app/clients/test_aws_sns.py index 7215c64b8..0a3db94c2 100644 --- a/tests/app/clients/test_aws_sns.py +++ b/tests/app/clients/test_aws_sns.py @@ -13,7 +13,10 @@ def test_send_sms_successful_returns_aws_sns_response(notify_api, mocker): boto_mock.publish.assert_called_once_with( PhoneNumber="+16135555555", Message=content, - MessageAttributes={'AWS.SNS.SMS.SMSType': {'DataType': 'String', 'StringValue': 'Transactional'}} + MessageAttributes={ + 'AWS.SNS.SMS.SMSType': {'DataType': 'String', 'StringValue': 'Transactional'}, + 'AWS.MM.SMS.OriginationNumber': {'DataType': 'String', 'StringValue': '+18446120782'} + } ) From dd0c7ebd568c4816ef78613c090fbf549ae7fee2 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Wed, 1 Mar 2023 13:46:08 -0500 Subject: [PATCH 3/7] Update sms sender numbers in db --- app/config.py | 2 +- .../versions/0346_notify_number_sms_sender.py | 4 +-- .../versions/0377_add_inbound_sms_number.py | 8 ++--- .../versions/0391_update_sms_numbers.py | 31 +++++++++++++++++++ 4 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 migrations/versions/0391_update_sms_numbers.py diff --git a/app/config.py b/app/config.py index cca2b5e60..2c29a6b7b 100644 --- a/app/config.py +++ b/app/config.py @@ -166,7 +166,7 @@ class Config(object): MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' - NOTIFY_INTERNATIONAL_SMS_SENDER = '18446120782' + NOTIFY_INTERNATIONAL_SMS_SENDER = getenv('AWS_US_TOLL_FREE_NUMBER') LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' # we only need real email in Live environment (production) diff --git a/migrations/versions/0346_notify_number_sms_sender.py b/migrations/versions/0346_notify_number_sms_sender.py index fca7ddbcd..b84f844ce 100644 --- a/migrations/versions/0346_notify_number_sms_sender.py +++ b/migrations/versions/0346_notify_number_sms_sender.py @@ -15,12 +15,12 @@ down_revision = '0345_move_broadcast_provider' SMS_SENDER_ID = 'd24b830b-57b4-4f14-bd80-02f46f8d54de' NOTIFY_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] -INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] +INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'].strip('+') def upgrade(): - sql = f"""INSERT INTO service_sms_senders (id, sms_sender, service_id, is_default, created_at) + sql = f"""INSERT INTO service_sms_senders (id, sms_sender, service_id, is_default, created_at) VALUES ('{SMS_SENDER_ID}', '{INBOUND_NUMBER}', '{NOTIFY_SERVICE_ID}',false, now())""" op.execute(sql) diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py index 0e808bae7..d06fceab0 100644 --- a/migrations/versions/0377_add_inbound_sms_number.py +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -15,7 +15,7 @@ revision = '0377_add_inbound_sms_number' down_revision = '0376_add_provider_response' INBOUND_NUMBER_ID = '9b5bc009-b847-4b1f-8a54-f3b5f95cff18' -INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] +INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'].strip('+') DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): @@ -26,20 +26,20 @@ def upgrade(): select_by_col = 'number' select_by_val = INBOUND_NUMBER op.execute(f"delete from {table_name} where {select_by_col} = '{select_by_val}'") - + # add the inbound number for the default service to inbound_numbers table_name = 'inbound_numbers' provider = 'sns' active = 'true' op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") - + # add the inbound number for the default service to service_sms_senders table_name = 'service_sms_senders' sms_sender = INBOUND_NUMBER select_by_col = 'id' select_by_val = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' op.execute(f"update {table_name} set {'sms_sender'}='{sms_sender}' where {select_by_col} = '{select_by_val}'") - + # add the inbound number for the default service to inbound_numbers table_name = 'service_permissions' permission = 'inbound_sms' diff --git a/migrations/versions/0391_update_sms_numbers.py b/migrations/versions/0391_update_sms_numbers.py new file mode 100644 index 000000000..0fceb33a9 --- /dev/null +++ b/migrations/versions/0391_update_sms_numbers.py @@ -0,0 +1,31 @@ +""" + +Revision ID: 0391_update_sms_numbers +Revises: 0390_drop_dvla_provider.py +Create Date: 2023-03-01 12:36:38.226954 + +""" +from alembic import op +from flask import current_app +import sqlalchemy as sa + + +revision = '0391_update_sms_numbers' +down_revision = '0390_drop_dvla_provider.py' +OLD_SMS_NUMBER = "18446120782" +NEW_SMS_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'].strip('+') + + +def upgrade(): + op.alter_column("service_sms_senders", "sms_sender", type_=sa.types.String(length=255)) + op.alter_column("inbound_numbers", "number", type_=sa.types.String(length=255)) + op.execute(f"UPDATE service_sms_senders SET sms_sender = '+{NEW_SMS_NUMBER}' WHERE sms_sender IN ('{OLD_SMS_NUMBER}', '{NEW_SMS_NUMBER}')") + op.execute(f"UPDATE inbound_numbers SET number = '+{NEW_SMS_NUMBER}' WHERE number IN ('{OLD_SMS_NUMBER}', '{NEW_SMS_NUMBER}')") + + + +def downgrade(): + op.execute(f"UPDATE service_sms_senders SET sms_sender = '{OLD_SMS_NUMBER}' WHERE sms_sender = '+{NEW_SMS_NUMBER}'") + op.execute(f"UPDATE inbound_numbers SET number = '{OLD_SMS_NUMBER}' WHERE number = '+{NEW_SMS_NUMBER}'") + op.alter_column("service_sms_senders", "sms_sender", type_=sa.types.String(length=11)) + op.alter_column("inbound_numbers", "number", type_=sa.types.String(length=11)) From 36975dda07e7f0e1825356aae33d55ed43c7356a Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 3 Mar 2023 13:58:26 -0500 Subject: [PATCH 4/7] Ensure CI runs have proper phone number format --- .github/workflows/checks.yml | 2 +- .github/workflows/daily_checks.yml | 2 +- .python-version | 1 + Pipfile.lock | 96 +++++++++++++++--------------- 4 files changed, 51 insertions(+), 50 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f4c8c38df..8347f3b77 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,7 +14,7 @@ env: WERKZEUG_DEBUG_PIN: off REDIS_ENABLED: 0 AWS_REGION: us-west-2 - AWS_US_TOLL_FREE_NUMBER: +18446120782 + AWS_US_TOLL_FREE_NUMBER: "+18446120782" jobs: build: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index a1c452e8e..b3d113b20 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -19,7 +19,7 @@ env: NOTIFY_EMAIL_DOMAIN: dispostable.com REDIS_ENABLED: 0 AWS_REGION: us-west-2 - AWS_US_TOLL_FREE_NUMBER: +18446120782 + AWS_US_TOLL_FREE_NUMBER: "+18446120782" jobs: pip-audit: diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..bd28b9c5c --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.9 diff --git a/Pipfile.lock b/Pipfile.lock index 1cd3ed3bd..6994a2662 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -277,32 +277,32 @@ }, "cryptography": { "hashes": [ - "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4", - "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f", - "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885", - "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502", - "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41", - "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965", - "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e", - "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc", - "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad", - "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505", - "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388", - "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6", - "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2", - "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef", - "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac", - "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695", - "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6", - "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336", - "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0", - "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c", - "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106", - "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a", - "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8" + "sha256:103e8f7155f3ce2ffa0049fe60169878d47a4364b277906386f8de21c9234aa1", + "sha256:23df8ca3f24699167daf3e23e51f7ba7334d504af63a94af468f468b975b7dd7", + "sha256:2725672bb53bb92dc7b4150d233cd4b8c59615cd8288d495eaa86db00d4e5c06", + "sha256:30b1d1bfd00f6fc80d11300a29f1d8ab2b8d9febb6ed4a38a76880ec564fae84", + "sha256:35d658536b0a4117c885728d1a7032bdc9a5974722ae298d6c533755a6ee3915", + "sha256:50cadb9b2f961757e712a9737ef33d89b8190c3ea34d0fb6675e00edbe35d074", + "sha256:5f8c682e736513db7d04349b4f6693690170f95aac449c56f97415c6980edef5", + "sha256:6236a9610c912b129610eb1a274bdc1350b5df834d124fa84729ebeaf7da42c3", + "sha256:788b3921d763ee35dfdb04248d0e3de11e3ca8eb22e2e48fef880c42e1f3c8f9", + "sha256:8bc0008ef798231fac03fe7d26e82d601d15bd16f3afaad1c6113771566570f3", + "sha256:8f35c17bd4faed2bc7797d2a66cbb4f986242ce2e30340ab832e5d99ae60e011", + "sha256:b49a88ff802e1993b7f749b1eeb31134f03c8d5c956e3c125c75558955cda536", + "sha256:bc0521cce2c1d541634b19f3ac661d7a64f9555135e9d8af3980965be717fd4a", + "sha256:bc5b871e977c8ee5a1bbc42fa8d19bcc08baf0c51cbf1586b0e87a2694dde42f", + "sha256:c43ac224aabcbf83a947eeb8b17eaf1547bce3767ee2d70093b461f31729a480", + "sha256:d15809e0dbdad486f4ad0979753518f47980020b7a34e9fc56e8be4f60702fac", + "sha256:d7d84a512a59f4412ca8549b01f94be4161c94efc598bf09d027d67826beddc0", + "sha256:e029b844c21116564b8b61216befabca4b500e6816fa9f0ba49527653cae2108", + "sha256:e8a0772016feeb106efd28d4a328e77dc2edae84dfbac06061319fdb669ff828", + "sha256:e944fe07b6f229f4c1a06a7ef906a19652bdd9fd54c761b0ff87e83ae7a30354", + "sha256:eb40fe69cfc6f5cdab9a5ebd022131ba21453cf7b8a7fd3631f45bbf52bed612", + "sha256:fa507318e427169ade4e9eccef39e9011cdc19534f55ca2f36ec3f388c1f70f3", + "sha256:ffd394c7896ed7821a6d13b24657c6a34b6e2650bd84ae063cf11ccffa4f1a97" ], "markers": "python_version >= '3.6'", - "version": "==39.0.1" + "version": "==39.0.2" }, "defusedxml": { "hashes": [ @@ -1604,32 +1604,32 @@ }, "cryptography": { "hashes": [ - "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4", - "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f", - "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885", - "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502", - "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41", - "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965", - "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e", - "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc", - "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad", - "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505", - "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388", - "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6", - "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2", - "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef", - "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac", - "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695", - "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6", - "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336", - "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0", - "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c", - "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106", - "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a", - "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8" + "sha256:103e8f7155f3ce2ffa0049fe60169878d47a4364b277906386f8de21c9234aa1", + "sha256:23df8ca3f24699167daf3e23e51f7ba7334d504af63a94af468f468b975b7dd7", + "sha256:2725672bb53bb92dc7b4150d233cd4b8c59615cd8288d495eaa86db00d4e5c06", + "sha256:30b1d1bfd00f6fc80d11300a29f1d8ab2b8d9febb6ed4a38a76880ec564fae84", + "sha256:35d658536b0a4117c885728d1a7032bdc9a5974722ae298d6c533755a6ee3915", + "sha256:50cadb9b2f961757e712a9737ef33d89b8190c3ea34d0fb6675e00edbe35d074", + "sha256:5f8c682e736513db7d04349b4f6693690170f95aac449c56f97415c6980edef5", + "sha256:6236a9610c912b129610eb1a274bdc1350b5df834d124fa84729ebeaf7da42c3", + "sha256:788b3921d763ee35dfdb04248d0e3de11e3ca8eb22e2e48fef880c42e1f3c8f9", + "sha256:8bc0008ef798231fac03fe7d26e82d601d15bd16f3afaad1c6113771566570f3", + "sha256:8f35c17bd4faed2bc7797d2a66cbb4f986242ce2e30340ab832e5d99ae60e011", + "sha256:b49a88ff802e1993b7f749b1eeb31134f03c8d5c956e3c125c75558955cda536", + "sha256:bc0521cce2c1d541634b19f3ac661d7a64f9555135e9d8af3980965be717fd4a", + "sha256:bc5b871e977c8ee5a1bbc42fa8d19bcc08baf0c51cbf1586b0e87a2694dde42f", + "sha256:c43ac224aabcbf83a947eeb8b17eaf1547bce3767ee2d70093b461f31729a480", + "sha256:d15809e0dbdad486f4ad0979753518f47980020b7a34e9fc56e8be4f60702fac", + "sha256:d7d84a512a59f4412ca8549b01f94be4161c94efc598bf09d027d67826beddc0", + "sha256:e029b844c21116564b8b61216befabca4b500e6816fa9f0ba49527653cae2108", + "sha256:e8a0772016feeb106efd28d4a328e77dc2edae84dfbac06061319fdb669ff828", + "sha256:e944fe07b6f229f4c1a06a7ef906a19652bdd9fd54c761b0ff87e83ae7a30354", + "sha256:eb40fe69cfc6f5cdab9a5ebd022131ba21453cf7b8a7fd3631f45bbf52bed612", + "sha256:fa507318e427169ade4e9eccef39e9011cdc19534f55ca2f36ec3f388c1f70f3", + "sha256:ffd394c7896ed7821a6d13b24657c6a34b6e2650bd84ae063cf11ccffa4f1a97" ], "markers": "python_version >= '3.6'", - "version": "==39.0.1" + "version": "==39.0.2" }, "cyclonedx-python-lib": { "hashes": [ From 40ec79e74cdb206e64b2bacb1f06ff18f3464732 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 3 Mar 2023 15:27:00 -0500 Subject: [PATCH 5/7] Only use service sender value if it is valid for SNS OriginationNumber --- app/clients/sms/aws_sns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/clients/sms/aws_sns.py b/app/clients/sms/aws_sns.py index 36ebbddfe..3f57c44c3 100644 --- a/app/clients/sms/aws_sns.py +++ b/app/clients/sms/aws_sns.py @@ -1,3 +1,4 @@ +import re from time import monotonic import botocore @@ -23,6 +24,7 @@ class AwsSnsClient(SmsClient): super(SmsClient, self).__init__(*args, **kwargs) self.current_app = current_app self.statsd_client = statsd_client + self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$") @property def name(self): @@ -31,6 +33,9 @@ class AwsSnsClient(SmsClient): def get_name(self): return self.name + def _valid_sender_number(self, sender): + return sender and re.match(self._valid_sender_regex, sender) + def send_sms(self, to, content, reference, sender=None, international=False): matched = False @@ -47,7 +52,7 @@ class AwsSnsClient(SmsClient): } } - if sender: + if self._valid_sender_number(sender): attributes["AWS.MM.SMS.OriginationNumber"] = { "DataType": "String", "StringValue": sender, From cb4ab8fb16e8e2e216740ea17c0f45cccb9b9940 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 3 Mar 2023 15:45:04 -0500 Subject: [PATCH 6/7] Remove obsolete references to AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY --- .github/workflows/deploy-demo.yml | 4 ---- .github/workflows/deploy.yml | 4 ---- docs/deploying.md | 2 +- manifest.yml | 2 -- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index a474ed8be..93f78fc2b 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -51,8 +51,6 @@ jobs: DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} SECRET_KEY: ${{ secrets.SECRET_KEY }} ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} with: cf_username: ${{ secrets.CLOUDGOV_USERNAME }} @@ -64,8 +62,6 @@ jobs: --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" - --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" - --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" - name: Check for changes to egress config diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 70a1c5c2c..4353e57b5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -56,8 +56,6 @@ jobs: DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} SECRET_KEY: ${{ secrets.SECRET_KEY }} ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} with: cf_username: ${{ secrets.CLOUDGOV_USERNAME }} @@ -69,8 +67,6 @@ jobs: --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" - --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" - --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" - name: Check for changes to egress config diff --git a/docs/deploying.md b/docs/deploying.md index 94d85af36..3d7b1ad4c 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -60,5 +60,5 @@ Rules for use: 1. start a pipenv shell as a shortcut to load `.env` file variables: `$ pipenv shell` 1. Deploy the application: ``` - cf push --vars-file deploy-config/sandbox.yml --var AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID --var AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + cf push --vars-file deploy-config/sandbox.yml ``` diff --git a/manifest.yml b/manifest.yml index 7db86dd06..448c70b81 100644 --- a/manifest.yml +++ b/manifest.yml @@ -48,8 +48,6 @@ applications: INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) - AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) - AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) AWS_US_TOLL_FREE_NUMBER: ((default_toll_free_number)) REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" From 22aa7e2787039068afdbdc03e267c919c4d659d0 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 3 Mar 2023 16:01:12 -0500 Subject: [PATCH 7/7] Remove default creds from s3 module --- app/aws/s3.py | 39 +++-------------------- tests/app/aws/test_s3.py | 69 ++++------------------------------------ 2 files changed, 11 insertions(+), 97 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 7a705c544..9d3b6dc9e 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -1,25 +1,19 @@ -import os - import botocore -from boto3 import Session, client +from boto3 import Session from flask import current_app FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' -default_access_key = os.environ.get('AWS_ACCESS_KEY_ID') -default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') -default_region = os.environ.get('AWS_REGION') - def get_s3_file( - bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region + bucket_name, file_location, access_key, secret_key, region ): s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region) return s3_file.get()['Body'].read().decode('utf-8') def get_s3_object( - bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region + bucket_name, file_location, access_key, secret_key, region ): session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region) s3 = session.resource('s3') @@ -27,7 +21,7 @@ def get_s3_object( def file_exists( - bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region + bucket_name, file_location, access_key, secret_key, region ): try: # try and access metadata of object @@ -85,28 +79,3 @@ def remove_contact_list_from_s3(service_id, contact_list_id): def remove_s3_object(bucket_name, object_key, access_key, secret_key, region): obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region) return obj.delete() - - -def get_list_of_files_by_suffix( - bucket_name, - subfolder='', - suffix='', - last_modified=None, - access_key=default_access_key, - secret_key=default_secret_key, - region=default_region -): - s3_client = client('s3', region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) - paginator = s3_client.get_paginator('list_objects_v2') - - page_iterator = paginator.paginate( - Bucket=bucket_name, - Prefix=subfolder - ) - - for page in page_iterator: - for obj in page.get('Contents', []): - key = obj['Key'] - if key.lower().endswith(suffix.lower()): - if not last_modified or obj['LastModified'] >= last_modified: - yield key diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index 56f26c5a0..735a7a141 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -1,17 +1,11 @@ -from datetime import datetime, timedelta +from datetime import datetime +from os import getenv -import pytest -import pytz -from freezegun import freeze_time +from app.aws.s3 import get_s3_file -from app.aws.s3 import ( - default_access_key, - default_region, - default_secret_key, - get_list_of_files_by_suffix, - get_s3_file, -) -from tests.app.conftest import datetime_in_past +default_access_key = getenv('AWS_ACCESS_KEY_ID') +default_secret_key = getenv('AWS_SECRET_ACCESS_KEY') +default_region = getenv('AWS_REGION') def single_s3_object_stub(key='foo', last_modified=None): @@ -24,7 +18,7 @@ def single_s3_object_stub(key='foo', last_modified=None): def test_get_s3_file_makes_correct_call(notify_api, mocker): get_s3_mock = mocker.patch('app.aws.s3.get_s3_object') - get_s3_file('foo-bucket', 'bar-file.txt') + get_s3_file('foo-bucket', 'bar-file.txt', default_access_key, default_secret_key, default_region) get_s3_mock.assert_called_with( 'foo-bucket', @@ -33,52 +27,3 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker): default_secret_key, default_region, ) - - -@freeze_time("2018-01-11 00:00:00") -@pytest.mark.parametrize('suffix_str, days_before, returned_no', [ - ('.ACK.txt', None, 1), - ('.ack.txt', None, 1), - ('.ACK.TXT', None, 1), - ('', None, 2), - ('', 1, 1), -]) -def test_get_list_of_files_by_suffix(notify_api, mocker, suffix_str, days_before, returned_no): - paginator_mock = mocker.patch('app.aws.s3.client') - multiple_pages_s3_object = [ - { - "Contents": [ - single_s3_object_stub('bar/foo.ACK.txt', datetime_in_past(1, 0)), - ] - }, - { - "Contents": [ - single_s3_object_stub('bar/foo1.rs.txt', datetime_in_past(2, 0)), - ] - } - ] - paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object - if (days_before): - key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix=suffix_str, - last_modified=datetime.now(tz=pytz.utc) - timedelta(days=days_before)) - else: - key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix=suffix_str) - - assert sum(1 for x in key) == returned_no - for k in key: - assert k == 'bar/foo.ACK.txt' - - -def test_get_list_of_files_by_suffix_empty_contents_return_with_no_error(notify_api, mocker): - paginator_mock = mocker.patch('app.aws.s3.client') - multiple_pages_s3_object = [ - { - "other_content": [ - 'some_values', - ] - } - ] - paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object - key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix='.pdf') - - assert sum(1 for x in key) == 0