From 17fec1c99e0a8e441d1887f2d4c53fa2cb0d7b37 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 28 May 2024 11:27:57 -0700 Subject: [PATCH] use moto to mock s3 --- notifications_utils/s3.py | 11 ++++ poetry.lock | 78 +++++++++++++++++++++++++++- pyproject.toml | 1 + tests/app/main/views/test_send.py | 14 +++++ tests/notifications_utils/test_s3.py | 2 + 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/notifications_utils/s3.py b/notifications_utils/s3.py index cdcc70a5c..a2886b381 100644 --- a/notifications_utils/s3.py +++ b/notifications_utils/s3.py @@ -38,6 +38,17 @@ def s3upload( region_name=region, ) _s3 = session.resource("s3", config=AWS_CLIENT_CONFIG) + # This 'proves' that use of moto in the relevant tests in test_send.py + # mocks everything related to S3. What you will see in the logs is: + # Exception: CREATED AT + # + # raise Exception(f"CREATED AT {_s3.Bucket(bucket_name).creation_date}") + if os.getenv("NOTIFY_ENVIRONMENT") == "test": + teststr = str(_s3.Bucket(bucket_name).creation_date).lower() + if "magicmock" not in teststr: + raise Exception( + f"xxxxxtest not mocked, use @mock_aws creation date is {teststr}" + ) key = _s3.Object(bucket_name, file_location) diff --git a/poetry.lock b/poetry.lock index 80a341491..5a32fc4d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1297,6 +1297,7 @@ files = [ {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"}, {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"}, + {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"}, {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"}, @@ -1551,6 +1552,50 @@ files = [ {file = "mistune-0.8.4.tar.gz", hash = "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e"}, ] +[[package]] +name = "moto" +version = "5.0.8" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "moto-5.0.8-py2.py3-none-any.whl", hash = "sha256:7d1035e366434bfa9fcc0621f07d5aa724b6846408071d540137a0554c46f214"}, + {file = "moto-5.0.8.tar.gz", hash = "sha256:517fb808dc718bcbdda54c6ffeaca0adc34cf6e10821bfb01216ce420a31765c"}, +] + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.14.0" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.15.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.5)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.5)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + [[package]] name = "msgpack" version = "1.0.8" @@ -1613,6 +1658,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2617,6 +2663,25 @@ requests = ">=2.22,<3" [package.extras] fixture = ["fixtures"] +[[package]] +name = "responses" +version = "0.25.0" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"}, + {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "rich" version = "13.7.1" @@ -2960,7 +3025,18 @@ files = [ {file = "xlwt-1.3.0.tar.gz", hash = "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88"}, ] +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "6c271d919c3736a844fa3674c1db0891e4c09378e6656b396ff60c594e34a862" +content-hash = "8f58d29f819ca160e10740e9774b5e528675208f0e8f2a51fa88ea0a62ea1dc8" diff --git a/pyproject.toml b/pyproject.toml index 2fc0b3c14..c0da02466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ flake8-print = "^5.0.0" flake8-pytest-style = "^1.7.2" isort = "^5.13.2" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} +moto="*" pip-audit = "*" pre-commit = "^3.7.1" pytest = "^8.2.1" diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 449259d8d..efaee4043 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -11,6 +11,7 @@ from zipfile import BadZipFile import pytest from flask import url_for +from moto import mock_aws from notifications_python_client.errors import HTTPError from xlrd.biffh import XLRDError from xlrd.xldate import XLDateAmbiguous, XLDateError, XLDateNegative, XLDateTooLarge @@ -426,6 +427,7 @@ def test_example_spreadsheet( list(zip(test_spreadsheet_files, repeat(True), repeat(302))) + list(zip(test_non_spreadsheet_files, repeat(False), repeat(200))), ) +@mock_aws def test_upload_files_in_different_formats( filename, acceptable_file, @@ -465,6 +467,7 @@ def test_upload_files_in_different_formats( ) +@mock_aws def test_send_messages_sanitises_and_truncates_file_name_for_metadata( client_request, service_one, @@ -572,6 +575,7 @@ def test_shows_error_if_parsing_exception( ) +@mock_aws def test_upload_csv_file_with_errors_shows_check_page_with_errors( client_request, service_one, @@ -619,6 +623,7 @@ def test_upload_csv_file_with_errors_shows_check_page_with_errors( assert "Upload your file again" in page.text +@mock_aws def test_upload_csv_file_with_empty_message_shows_check_page_with_errors( client_request, service_one, @@ -671,6 +676,7 @@ def test_upload_csv_file_with_empty_message_shows_check_page_with_errors( assert page.select("tbody tr td")[1]["colspan"] == "2" +@mock_aws def test_upload_csv_file_with_very_long_placeholder_shows_check_page_with_errors( client_request, service_one, @@ -807,6 +813,7 @@ def test_upload_csv_file_with_very_long_placeholder_shows_check_page_with_errors ), ], ) +@mock_aws def test_upload_csv_file_with_missing_columns_shows_error( client_request, mocker, @@ -882,6 +889,7 @@ def test_upload_csv_size_too_big( assert "File must be smaller than 10Mb" in page.text +@mock_aws def test_upload_valid_csv_redirects_to_check_page( client_request, mock_get_service_template_with_placeholders, @@ -928,6 +936,7 @@ def test_upload_valid_csv_redirects_to_check_page( ), ], ) +@mock_aws def test_upload_valid_csv_shows_preview_and_table( client_request, mocker, @@ -1021,6 +1030,7 @@ def test_upload_valid_csv_shows_preview_and_table( assert normalize_spaces(str(row.select("td")[index])) == cell +@mock_aws def test_show_all_columns_if_there_are_duplicate_recipient_columns( client_request, mocker, @@ -1071,6 +1081,7 @@ def test_show_all_columns_if_there_are_duplicate_recipient_columns( (5, 404), ], ) +@mock_aws def test_404_for_previewing_a_row_out_of_range( client_request, mocker, @@ -1519,6 +1530,7 @@ def test_send_one_off_redirects_to_end_if_step_out_of_bounds( create_active_caseworking_user(), ], ) +@mock_aws def test_send_one_off_redirects_to_start_if_you_skip_steps( client_request, service_one, @@ -1623,6 +1635,7 @@ def test_send_one_off_sms_message_redirects( create_active_caseworking_user(), ], ) +@mock_aws def test_send_one_off_email_to_self_without_placeholders_redirects_to_check_page( client_request, mocker, @@ -1828,6 +1841,7 @@ def test_download_example_csv( assert "text/csv" in response.headers["Content-Type"] +@mock_aws def test_upload_csvfile_with_valid_phone_shows_all_numbers( client_request, mock_get_service_template, diff --git a/tests/notifications_utils/test_s3.py b/tests/notifications_utils/test_s3.py index 46b863c4f..d05fa8fdc 100644 --- a/tests/notifications_utils/test_s3.py +++ b/tests/notifications_utils/test_s3.py @@ -2,6 +2,7 @@ from urllib.parse import parse_qs import botocore import pytest +from moto import mock_aws from notifications_utils.s3 import S3ObjectNotFound, s3download, s3upload @@ -12,6 +13,7 @@ location = "some_file_location" content_type = "binary/octet-stream" +@mock_aws def test_s3upload_save_file_to_bucket(mocker): mocked = mocker.patch("notifications_utils.s3.Session.resource") s3upload(