Merge branch 'main' into jskinne3-upgrade-and-protect-production-env

This commit is contained in:
John Skiles Skinner
2024-06-24 10:53:49 -07:00
committed by GitHub
15 changed files with 97 additions and 106 deletions

View File

@@ -9,10 +9,10 @@ runs:
sudo apt-get update \
&& sudo apt-get install -y --no-install-recommends \
libcurl4-openssl-dev
- name: Set up Python 3.12
- name: Set up Python 3.12.3
uses: actions/setup-python@v4
with:
python-version: "3.12"
python-version: "3.12.3"
- name: Install poetry
shell: bash
run: pip install --upgrade poetry

View File

@@ -89,7 +89,7 @@ def send_sms_to_provider(notification):
except Exception:
# It is our 2facode, maybe
key = f"2facode-{notification.id}".replace(" ", "")
recipient = redis_store.raw_get(key)
recipient = redis_store.get(key)
if recipient:
recipient = recipient.decode("utf-8")

View File

@@ -361,7 +361,7 @@ def create_2fa_code(
saved_notification.personalisation = personalisation
key = f"2facode-{saved_notification.id}".replace(" ", "")
recipient = str(recipient)
redis_store.raw_set(key, recipient, ex=60 * 60)
redis_store.set(key, recipient, ex=60 * 60)
# Assume that we never want to observe the Notify service's research mode
# setting for this notification - we still need to be able to log into the

View File

@@ -1,8 +1,8 @@
env: production
web_instances: 2
web_memory: 1G
web_memory: 2G
worker_instances: 1
worker_memory: 1G
worker_memory: 2G
scheduler_memory: 256M
public_api_route: notify-api.app.cloud.gov
admin_base_url: https://beta.notify.gov

View File

@@ -133,19 +133,13 @@ class RedisClient:
else:
return False
def raw_set(self, key, value, ex=None, px=None, nx=False, xx=False):
self.redis_store.set(key, value, ex, px, nx, xx)
def set(
self, key, value, ex=None, px=None, nx=False, xx=False, raise_exception=False
):
key = prepare_value(key)
value = prepare_value(value)
if self.active:
try:
self.redis_store.set(key, value, ex, px, nx, xx)
except Exception as e:
self.__handle_exception(e, raise_exception, "set", key)
self.redis_store.set(key, value, ex, px, nx, xx)
def incr(self, key, raise_exception=False):
key = prepare_value(key)
@@ -155,16 +149,10 @@ class RedisClient:
except Exception as e:
self.__handle_exception(e, raise_exception, "incr", key)
def raw_get(self, key):
return self.redis_store.get(key)
def get(self, key, raise_exception=False):
key = prepare_value(key)
if self.active:
try:
return self.redis_store.get(key)
except Exception as e:
self.__handle_exception(e, raise_exception, "get", key)
return self.redis_store.get(key)
return None

16
poetry.lock generated
View File

@@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
[[package]]
name = "aiohttp"
@@ -2098,13 +2098,9 @@ files = [
{file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},
{file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},
{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},
{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},
{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},
{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},
{file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},
{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},
{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},
{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},
{file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},
{file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},
{file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},
@@ -2493,6 +2489,7 @@ files = [
{file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
{file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
{file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
{file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
{file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
]
@@ -2826,13 +2823,13 @@ ptyprocess = ">=0.5"
[[package]]
name = "phonenumbers"
version = "8.13.38"
version = "8.13.39"
description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
optional = false
python-versions = "*"
files = [
{file = "phonenumbers-8.13.38-py2.py3-none-any.whl", hash = "sha256:d22aa747fb591ef2a18afec13cab5a0e294ab20fce5a1560e4949e459e70eeef"},
{file = "phonenumbers-8.13.38.tar.gz", hash = "sha256:2822c74ee9334e9d8ad792fc352cc8d21004307349b6b1bb61da12937fa2eaba"},
{file = "phonenumbers-8.13.39-py2.py3-none-any.whl", hash = "sha256:3ad2d086fa71e7eef409001b9195ac54bebb0c6e3e752209b558ca192c9229a0"},
{file = "phonenumbers-8.13.39.tar.gz", hash = "sha256:db7ca4970d206b2056231105300753b1a5b229f43416f8c2b3010e63fbb68d77"},
]
[[package]]
@@ -3475,7 +3472,6 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -4752,4 +4748,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.12.2"
content-hash = "460acc2fefc9038bbbf2290afb1c657ea43018dd8abb626370044988d49744d6"
content-hash = "122fe6459adad8b8105d12739e4db159ae39fc32e969d02c7c24436ed3fb85b1"

View File

@@ -55,7 +55,7 @@ geojson = "^3.1.0"
govuk-bank-holidays = "^0.14"
numpy = "^1.26.4"
ordered-set = "^4.1.0"
phonenumbers = "^8.13.38"
phonenumbers = "^8.13.39"
python-json-logger = "^2.0.7"
pytz = "^2024.1"
regex = "^2024.5.15"

View File

@@ -1 +1 @@
python-3.12.x
python-3.12.3

View File

@@ -3,27 +3,33 @@ locals {
cf_space_name = "notify-demo"
env = "demo"
app_name = "notify-api"
recursive_delete = false
recursive_delete = false # deprecated, still used in shared modules
}
resource "null_resource" "prevent_destroy" {
lifecycle {
prevent_destroy = true
}
}
module "database" {
source = "github.com/18f/terraform-cloudgov//database?ref=v0.7.1"
source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-rds-${local.env}"
recursive_delete = local.recursive_delete
rds_plan_name = "micro-psql"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-rds-${local.env}"
rds_plan_name = "micro-psql"
}
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-${local.env}"
recursive_delete = local.recursive_delete
redis_plan_name = "redis-dev"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-${local.env}"
redis_plan_name = "redis-dev"
}
module "redis-v70" {
@@ -41,12 +47,11 @@ module "redis-v70" {
}
module "csv_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
recursive_delete = local.recursive_delete
name = "${local.app_name}-csv-upload-bucket-${local.env}"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-csv-upload-bucket-${local.env}"
}
module "egress-space" {
@@ -66,7 +71,6 @@ module "ses_email" {
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-ses-${local.env}"
recursive_delete = local.recursive_delete
aws_region = "us-west-2"
email_domain = "notify.sandbox.10x.gsa.gov"
email_receipt_error = "notify-support@gsa.gov"
@@ -78,7 +82,6 @@ module "sns_sms" {
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-sns-${local.env}"
recursive_delete = local.recursive_delete
aws_region = "us-east-1"
monthly_spend_limit = 25
}

View File

@@ -22,7 +22,7 @@ module "database" {
rds_plan_name = "small-psql-redundant"
}
module "redis" {
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
@@ -31,6 +31,20 @@ module "redis" {
redis_plan_name = "redis-3node-large"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-dev"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "csv_upload_bucket" {
source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0"

View File

@@ -16,19 +16,19 @@ resource "null_resource" "prevent_destroy" {
module "database" {
source = "github.com/GSA-TTS/terraform-cloudgov//database?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-rds-${local.env}"
rds_plan_name = "micro-psql"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-rds-${local.env}"
rds_plan_name = "micro-psql"
}
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-${local.env}"
redis_plan_name = "redis-dev"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-${local.env}"
redis_plan_name = "redis-dev"
}
module "redis-v70" {
@@ -48,9 +48,9 @@ module "redis-v70" {
module "csv_upload_bucket" {
source = "github.com/GSA-TTS/terraform-cloudgov//s3?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-csv-upload-bucket-${local.env}"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-csv-upload-bucket-${local.env}"
}
module "egress-space" {

View File

@@ -16,7 +16,7 @@ module "database" {
rds_plan_name = "micro-psql"
}
module "redis" {
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
cf_org_name = local.cf_org_name
@@ -26,6 +26,20 @@ module "redis" {
redis_plan_name = "redis-dev"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-dev"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "csv_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"

View File

@@ -31,8 +31,8 @@ def test_create_invited_user(
extra_args,
expected_start_of_invite_url,
):
mocker.patch("app.service_invite.rest.redis_store.raw_set")
mocker.patch("app.service_invite.rest.redis_store.raw_get")
mocker.patch("app.service_invite.rest.redis_store.set")
mocker.patch("app.service_invite.rest.redis_store.get")
mocked = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
email_address = "invited_user@service.gov.uk"
@@ -96,8 +96,8 @@ def test_create_invited_user_without_auth_type(
admin_request, sample_service, mocker, invitation_email_template
):
mocker.patch("app.service_invite.rest.redis_store.raw_set")
mocker.patch("app.service_invite.rest.redis_store.raw_get")
mocker.patch("app.service_invite.rest.redis_store.set")
mocker.patch("app.service_invite.rest.redis_store.get")
mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
email_address = "invited_user@service.gov.uk"
invite_from = sample_service.users[0]
@@ -220,8 +220,8 @@ def test_resend_expired_invite(
mocker,
):
mocker.patch("app.service_invite.rest.redis_store.raw_set")
mocker.patch("app.service_invite.rest.redis_store.raw_get")
mocker.patch("app.service_invite.rest.redis_store.set")
mocker.patch("app.service_invite.rest.redis_store.get")
url = f"/service/{sample_expired_user.service_id}/invite/{sample_expired_user.id}/resend"
mock_send = mocker.patch("app.service_invite.rest.send_notification_to_queue")
mock_persist = mocker.patch("app.service_invite.rest.persist_notification")

View File

@@ -201,10 +201,10 @@ def test_send_user_sms_code(client, sample_user, sms_code_template, mocker):
"""
notify_service = dao_fetch_service_by_id(current_app.config["NOTIFY_SERVICE_ID"])
mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get")
mock_redis_get = mocker.patch("app.user.rest.redis_store.get")
mock_redis_get.return_value = "foo"
mocker.patch("app.user.rest.redis_store.raw_set")
mocker.patch("app.user.rest.redis_store.set")
auth_header = create_admin_authorization_header()
mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111")
mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async")
@@ -242,10 +242,10 @@ def test_send_user_code_for_sms_with_optional_to_field(
Tests POST endpoint /user/<user_id>/sms-code with optional to field
"""
mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get")
mock_redis_get = mocker.patch("app.user.rest.redis_store.get")
mock_redis_get.return_value = "foo"
mocker.patch("app.user.rest.redis_store.raw_set")
mocker.patch("app.user.rest.redis_store.set")
to_number = "+14254147755"
mocked = mocker.patch("app.user.rest.create_secret_code", return_value="11111")
mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async")
@@ -469,10 +469,10 @@ def test_send_user_email_code(
deliver_email = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
sample_user.auth_type = auth_type
mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get")
mock_redis_get = mocker.patch("app.user.rest.redis_store.get")
mock_redis_get.return_value = "foo"
mocker.patch("app.user.rest.redis_store.raw_set")
mocker.patch("app.user.rest.redis_store.set")
admin_request.post(
"user.send_user_2fa_code",
@@ -497,10 +497,10 @@ def test_send_user_email_code_with_urlencoded_next_param(
):
mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.raw_get")
mock_redis_get = mocker.patch("app.celery.scheduled_tasks.redis_store.get")
mock_redis_get.return_value = "foo"
mocker.patch("app.celery.scheduled_tasks.redis_store.raw_set")
mocker.patch("app.celery.scheduled_tasks.redis_store.set")
data = {"to": None, "next": "/services"}
admin_request.post(
@@ -581,10 +581,10 @@ def test_user_verify_email_code_fails_if_code_already_used(
def test_send_user_2fa_code_sends_from_number_for_international_numbers(
client, sample_user, mocker, sms_code_template
):
mock_redis_get = mocker.patch("app.user.rest.redis_store.raw_get")
mock_redis_get = mocker.patch("app.user.rest.redis_store.get")
mock_redis_get.return_value = "foo"
mocker.patch("app.user.rest.redis_store.raw_set")
mocker.patch("app.user.rest.redis_store.set")
sample_user.mobile_number = "+601117224412"
auth_header = create_admin_authorization_header()

View File

@@ -1,5 +1,5 @@
import uuid
from unittest.mock import Mock, call
from unittest.mock import Mock
import pytest
from freezegun import freeze_time
@@ -59,30 +59,6 @@ def failing_redis_client(mocked_redis_client, delete_mock):
return mocked_redis_client
def test_should_not_raise_exception_if_raise_set_to_false(
app, caplog, failing_redis_client, mocker
):
mock_logger = mocker.patch("flask.Flask.logger")
assert failing_redis_client.get("get_key") is None
assert failing_redis_client.set("set_key", "set_value") is None
assert failing_redis_client.incr("incr_key") is None
assert failing_redis_client.exceeded_rate_limit("rate_limit_key", 100, 100) is False
assert failing_redis_client.delete("delete_key") is None
assert failing_redis_client.delete("a", "b", "c") is None
assert failing_redis_client.delete_by_pattern("pattern") == 0
assert mock_logger.mock_calls == [
call.exception("Redis error performing get on get_key"),
call.exception("Redis error performing set on set_key"),
call.exception("Redis error performing incr on incr_key"),
call.exception("Redis error performing rate-limit-pipeline on rate_limit_key"),
call.exception("Redis error performing delete on delete_key"),
call.exception("Redis error performing delete on a, b, c"),
call.exception("Redis error performing delete-by-pattern on pattern"),
]
def test_should_raise_exception_if_raise_set_to_true(
app,
failing_redis_client,