use moto to mock s3

notifications_utils/s3.py

@@ -38,6 +38,17 @@ def s3upload(
         region_name=region,
     )
     _s3 = session.resource("s3", config=AWS_CLIENT_CONFIG)
+    # This 'proves' that use of moto in the relevant tests in test_send.py
+    # mocks everything related to S3.  What you will see in the logs is:
+    # Exception: CREATED AT <MagicMock name='resource().Bucket().creation_date' id='4665562448'>
+    #
+    # raise Exception(f"CREATED AT {_s3.Bucket(bucket_name).creation_date}")
+    if os.getenv("NOTIFY_ENVIRONMENT") == "test":
+        teststr = str(_s3.Bucket(bucket_name).creation_date).lower()
+        if "magicmock" not in teststr:
+            raise Exception(
+                f"xxxxxtest not mocked, use @mock_aws creation date is {teststr}"
+            )
 
     key = _s3.Object(bucket_name, file_location)
 

poetry.lock

@@ -1297,6 +1297,7 @@ files = [
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"},
+    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"},
@@ -1551,6 +1552,50 @@ files = [
     {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"},
 ]
 
+[[package]]
+name = "moto"
+version = "5.0.8"
+description = ""
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "moto-5.0.8-py2.py3-none-any.whl", hash = "sha256:7d1035e366434bfa9fcc0621f07d5aa724b6846408071d540137a0554c46f214"},
+    {file = "moto-5.0.8.tar.gz", hash = "sha256:517fb808dc718bcbdda54c6ffeaca0adc34cf6e10821bfb01216ce420a31765c"},
+]
+
+[package.dependencies]
+boto3 = ">=1.9.201"
+botocore = ">=1.14.0"
+cryptography = ">=3.3.1"
+Jinja2 = ">=2.10.1"
+python-dateutil = ">=2.1,<3.0.0"
+requests = ">=2.5"
+responses = ">=0.15.0"
+werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1"
+xmltodict = "*"
+
+[package.extras]
+all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"]
+apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"]
+apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"]
+appsync = ["graphql-core"]
+awslambda = ["docker (>=3.0.0)"]
+batch = ["docker (>=3.0.0)"]
+cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"]
+cognitoidp = ["joserfc (>=0.9.0)"]
+dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"]
+dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"]
+glue = ["pyparsing (>=3.0.7)"]
+iotdata = ["jsondiff (>=1.1.2)"]
+proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"]
+resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)"]
+s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.5)"]
+s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.5)"]
+server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"]
+ssm = ["PyYAML (>=5.1)"]
+stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"]
+xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"]
+
 [[package]]
 name = "msgpack"
 version = "1.0.8"
@@ -1613,6 +1658,7 @@ files = [
     {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
     {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
     {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
+    {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
     {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
 ]
 
@@ -2617,6 +2663,25 @@ requests = ">=2.22,<3"
 [package.extras]
 fixture = ["fixtures"]
 
+[[package]]
+name = "responses"
+version = "0.25.0"
+description = "A utility library for mocking out the `requests` Python library."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"},
+    {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"},
+]
+
+[package.dependencies]
+pyyaml = "*"
+requests = ">=2.30.0,<3.0"
+urllib3 = ">=1.25.10,<3.0"
+
+[package.extras]
+tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"]
+
 [[package]]
 name = "rich"
 version = "13.7.1"
@@ -2960,7 +3025,18 @@ files = [
     {file = "xlwt-1.3.0.tar.gz", hash = "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88"},
 ]
 
+[[package]]
+name = "xmltodict"
+version = "0.13.0"
+description = "Makes working with XML feel like you are working with JSON"
+optional = false
+python-versions = ">=3.4"
+files = [
+    {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"},
+    {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
+]
+
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12.2"
-content-hash = "6c271d919c3736a844fa3674c1db0891e4c09378e6656b396ff60c594e34a862"
+content-hash = "8f58d29f819ca160e10740e9774b5e528675208f0e8f2a51fa88ea0a62ea1dc8"

pyproject.toml

@@ -82,6 +82,7 @@ flake8-print = "^5.0.0"
 flake8-pytest-style = "^1.7.2"
 isort = "^5.13.2"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
+moto="*"
 pip-audit = "*"
 pre-commit = "^3.7.1"
 pytest = "^8.2.1"

tests/app/main/views/test_send.py

@@ -11,6 +11,7 @@ from zipfile import BadZipFile
 
 import pytest
 from flask import url_for
+from moto import mock_aws
 from notifications_python_client.errors import HTTPError
 from xlrd.biffh import XLRDError
 from xlrd.xldate import XLDateAmbiguous, XLDateError, XLDateNegative, XLDateTooLarge
@@ -426,6 +427,7 @@ def test_example_spreadsheet(
     list(zip(test_spreadsheet_files, repeat(True), repeat(302)))
     + list(zip(test_non_spreadsheet_files, repeat(False), repeat(200))),
 )
+@mock_aws
 def test_upload_files_in_different_formats(
     filename,
     acceptable_file,
@@ -465,6 +467,7 @@ def test_upload_files_in_different_formats(
         )
 
 
+@mock_aws
 def test_send_messages_sanitises_and_truncates_file_name_for_metadata(
     client_request,
     service_one,
@@ -572,6 +575,7 @@ def test_shows_error_if_parsing_exception(
     )
 
 
+@mock_aws
 def test_upload_csv_file_with_errors_shows_check_page_with_errors(
     client_request,
     service_one,
@@ -619,6 +623,7 @@ def test_upload_csv_file_with_errors_shows_check_page_with_errors(
     assert "Upload your file again" in page.text
 
 
+@mock_aws
 def test_upload_csv_file_with_empty_message_shows_check_page_with_errors(
     client_request,
     service_one,
@@ -671,6 +676,7 @@ def test_upload_csv_file_with_empty_message_shows_check_page_with_errors(
     assert page.select("tbody tr td")[1]["colspan"] == "2"
 
 
+@mock_aws
 def test_upload_csv_file_with_very_long_placeholder_shows_check_page_with_errors(
     client_request,
     service_one,
@@ -807,6 +813,7 @@ def test_upload_csv_file_with_very_long_placeholder_shows_check_page_with_errors
         ),
     ],
 )
+@mock_aws
 def test_upload_csv_file_with_missing_columns_shows_error(
     client_request,
     mocker,
@@ -882,6 +889,7 @@ def test_upload_csv_size_too_big(
     assert "File must be smaller than 10Mb" in page.text
 
 
+@mock_aws
 def test_upload_valid_csv_redirects_to_check_page(
     client_request,
     mock_get_service_template_with_placeholders,
@@ -928,6 +936,7 @@ def test_upload_valid_csv_redirects_to_check_page(
         ),
     ],
 )
+@mock_aws
 def test_upload_valid_csv_shows_preview_and_table(
     client_request,
     mocker,
@@ -1021,6 +1030,7 @@ def test_upload_valid_csv_shows_preview_and_table(
             assert normalize_spaces(str(row.select("td")[index])) == cell
 
 
+@mock_aws
 def test_show_all_columns_if_there_are_duplicate_recipient_columns(
     client_request,
     mocker,
@@ -1071,6 +1081,7 @@ def test_show_all_columns_if_there_are_duplicate_recipient_columns(
         (5, 404),
     ],
 )
+@mock_aws
 def test_404_for_previewing_a_row_out_of_range(
     client_request,
     mocker,
@@ -1519,6 +1530,7 @@ def test_send_one_off_redirects_to_end_if_step_out_of_bounds(
         create_active_caseworking_user(),
     ],
 )
+@mock_aws
 def test_send_one_off_redirects_to_start_if_you_skip_steps(
     client_request,
     service_one,
@@ -1623,6 +1635,7 @@ def test_send_one_off_sms_message_redirects(
         create_active_caseworking_user(),
     ],
 )
+@mock_aws
 def test_send_one_off_email_to_self_without_placeholders_redirects_to_check_page(
     client_request,
     mocker,
@@ -1828,6 +1841,7 @@ def test_download_example_csv(
     assert "text/csv" in response.headers["Content-Type"]
 
 
+@mock_aws
 def test_upload_csvfile_with_valid_phone_shows_all_numbers(
     client_request,
     mock_get_service_template,

tests/notifications_utils/test_s3.py

@@ -2,6 +2,7 @@ from urllib.parse import parse_qs
 
 import botocore
 import pytest
+from moto import mock_aws
 
 from notifications_utils.s3 import S3ObjectNotFound, s3download, s3upload
 
@@ -12,6 +13,7 @@ location = "some_file_location"
 content_type = "binary/octet-stream"
 
 
+@mock_aws
 def test_s3upload_save_file_to_bucket(mocker):
     mocked = mocker.patch("notifications_utils.s3.Session.resource")
     s3upload(