diff --git a/.profile b/.profile index 1c1f98b4b..f7721ea96 100644 --- a/.profile +++ b/.profile @@ -3,5 +3,6 @@ # https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile ## +export http_proxy=$egress_proxy export https_proxy=$egress_proxy export NEW_RELIC_PROXY_HOST=$egress_proxy diff --git a/app/aws/s3.py b/app/aws/s3.py index 700f252c2..e226f05a1 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -11,7 +11,8 @@ from app.clients import AWS_CLIENT_CONFIG FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv" -JOBS = ExpiringDict(max_len=100, max_age_seconds=3600 * 4) +JOBS = ExpiringDict(max_len=1000, max_age_seconds=3600 * 4) + JOBS_CACHE_HITS = "JOBS_CACHE_HITS" JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES" @@ -93,7 +94,33 @@ def incr_jobs_cache_hits(): redis_store.incr(JOBS_CACHE_HITS) hits = redis_store.get(JOBS_CACHE_HITS).decode("utf-8") misses = redis_store.get(JOBS_CACHE_MISSES).decode("utf-8") - current_app.logger.info(f"JOBS CACHE MISS hits {hits} misses {misses}") + current_app.logger.info(f"JOBS CACHE HIT hits {hits} misses {misses}") + + +def extract_phones(job): + job = job.split("\r\n") + first_row = job[0] + job.pop(0) + first_row = first_row.split(",") + phone_index = 0 + for item in first_row: + if item.lower() == "phone number": + break + phone_index = phone_index + 1 + phones = {} + job_row = 0 + for row in job: + row = row.split(",") + phone_index = 0 + for item in first_row: + if item.lower() == "phone number": + break + phone_index = phone_index + 1 + my_phone = row[phone_index] + my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone) + phones[job_row] = my_phone + job_row = job_row + 1 + return phones def get_phone_number_from_s3(service_id, job_id, job_row_number): @@ -108,25 +135,37 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number): else: incr_jobs_cache_hits() - job = job.split("\r\n") - first_row = job[0] - job.pop(0) - first_row = first_row.split(",") - phone_index = 0 - for item in first_row: - if item.lower() == "phone number": - break - phone_index = phone_index + 1 - - correct_row = job[job_row_number] - correct_row = correct_row.split(",") - - # This could happen if an old job cannot be retrieved from s3 - if len(correct_row) <= phone_index: + # If the job is None after our attempt to retrieve it from s3, it + # probably means the job is old and has been deleted from s3, in + # which case there is nothing we can do. It's unlikely to run into + # this, but it could theoretically happen, especially if we ever + # change the task schedules + if job is None: + current_app.logger.warning( + "Couldnt find phone for job_id {job_id} row number {job_row_number} because job is missing" + ) + return "Unknown Phone" + + # If we look in the JOBS cache for the quick lookup dictionary of phones for a given job + # and that dictionary is not there, create it + if JOBS.get(f"{job_id}_phones") is None: + JOBS[f"{job_id}_phones"] = extract_phones(job) + + # If we can find the quick dictionary, use it + if JOBS.get(f"{job_id}_phones") is not None: + phone_to_return = JOBS.get(f"{job_id}_phones").get(job_row_number) + if phone_to_return: + return phone_to_return + else: + current_app.logger.warning( + f"Was unable to retrieve phone number from lookup dictionary for job {job_id}" + ) + return "Unknown Phone" + else: + current_app.logger.error( + f"Was unable to construct lookup dictionary for job {job_id}" + ) return "Unknown Phone" - my_phone = correct_row[phone_index] - my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone) - return my_phone def get_personalisation_from_s3(service_id, job_id, job_row_number): diff --git a/app/service/rest.py b/app/service/rest.py index 7c2dfdc3f..f44e9c8a1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -440,6 +440,9 @@ def get_all_notifications_for_service(service_id): ) notification.to = recipient notification.normalised_to = recipient + else: + notification.to = "1" + notification.normalised_to = "1" kwargs = request.args.to_dict() kwargs["service_id"] = service_id @@ -479,7 +482,7 @@ def get_all_notifications_for_service(service_id): page, len(next_page_of_pagination.items), ".get_all_notifications_for_service", - **kwargs + **kwargs, ) if count_pages else {}, diff --git a/deploy-config/egress_proxy/notify-api-demo.allow.acl b/deploy-config/egress_proxy/notify-api-demo.allow.acl index 36a93c46f..6aa6aa2b4 100644 --- a/deploy-config/egress_proxy/notify-api-demo.allow.acl +++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl @@ -8,3 +8,5 @@ s3-fips.us-west-2.amazonaws.com sns-fips.us-east-1.amazonaws.com gov-collector.newrelic.com egress-proxy-notify-api-demo.apps.internal +idp.int.identitysandbox.gov +secure.login.gov diff --git a/deploy-config/egress_proxy/notify-api-production.allow.acl b/deploy-config/egress_proxy/notify-api-production.allow.acl index 2cc1bd8fe..a7a70636f 100644 --- a/deploy-config/egress_proxy/notify-api-production.allow.acl +++ b/deploy-config/egress_proxy/notify-api-production.allow.acl @@ -7,3 +7,5 @@ s3-fips.us-gov-west-1.amazonaws.com sns.us-gov-west-1.amazonaws.com gov-collector.newrelic.com egress-proxy-notify-api-production.apps.internal +idp.int.identitysandbox.gov +secure.login.gov diff --git a/deploy-config/egress_proxy/notify-api-staging.allow.acl b/deploy-config/egress_proxy/notify-api-staging.allow.acl index 3768c3c74..eb46ce948 100644 --- a/deploy-config/egress_proxy/notify-api-staging.allow.acl +++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl @@ -8,3 +8,5 @@ s3-fips.us-west-2.amazonaws.com sns-fips.us-west-2.amazonaws.com gov-collector.newrelic.com egress-proxy-notify-api-staging.apps.internal +idp.int.identitysandbox.gov +secure.login.gov diff --git a/poetry.lock b/poetry.lock index 406d20bed..e850898f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -280,19 +280,22 @@ typecheck = ["mypy"] [[package]] name = "beautifulsoup4" -version = "4.12.2" +version = "4.12.3" description = "Screen-scraping library" optional = false python-versions = ">=3.6.0" files = [ - {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, - {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, ] [package.dependencies] soupsieve = ">1.2" [package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] @@ -4720,4 +4723,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "3389aa4ce5477dd99ab467168569e9920b942777f37e671bceb8e362ca362f76" +content-hash = "a6d1d609f32455bcd4a1b2934512bba368d3bad792a032bb0e8e2b49f6f5f99a" diff --git a/pyproject.toml b/pyproject.toml index bec3af860..95bf20635 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" python = ">=3.9,<3.12" alembic = "==1.13.1" amqp = "==5.2.0" -beautifulsoup4 = "==4.12.2" +beautifulsoup4 = "==4.12.3" boto3 = "^1.29.6" botocore = "^1.32.6" cachetools = "==5.3.2"