From 7b03d02447f4e1783a4fa3ea1e353e50f1f28a44 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 6 Aug 2024 11:31:05 -0700 Subject: [PATCH 01/13] add front end for download all users --- app/main/views/platform_admin.py | 25 ++++++++++++++++++- app/navigation.py | 1 + app/notify_client/user_api_client.py | 4 +++ .../views/platform-admin/reports.html | 4 +++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 6190ac988..07e973704 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -3,7 +3,7 @@ import json from collections import OrderedDict from datetime import datetime -from flask import abort, flash, render_template, request, url_for +from flask import Response, abort, flash, render_template, request, send_file, url_for from notifications_python_client.errors import HTTPError from app import ( @@ -70,6 +70,29 @@ def platform_admin(): ) +@main.route("/platform-admin/download-all-users") +@user_is_platform_admin +def download_all_users(): + + # Create a CSV string from the user data + csv_data = "Name,Email Address,Mobile Number,Service\n" + users = user_api_client.get_all_users_detailed() + + if len(users) == 0: + return "No data to download." + users = json.loads(users) + for user in users: + if user["name"].startswith("e2e"): + continue + csv_data += f"{user['name']},{user['email_address']},{user['mobile_number']},{user['service']}\n" + + # Create a direct download response with the CSV data and appropriate headers + response = Response(csv_data, content_type="text/csv") + response.headers["Content-Disposition"] = "attachment; filename=users.csv" + + return response + + def is_over_threshold(number, total, threshold): percentage = number / total * 100 if total else 0 return percentage > threshold diff --git a/app/navigation.py b/app/navigation.py index 3ce3b6e62..6ef0907a6 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -123,6 +123,7 @@ class HeaderNavigation(Navigation): "get_billing_report", "get_users_report", "get_daily_volumes", + "download_all_users", "get_daily_sms_provider_volumes", "get_volumes_by_service", "organizations", diff --git a/app/notify_client/user_api_client.py b/app/notify_client/user_api_client.py index 06d6385b2..e47bc4030 100644 --- a/app/notify_client/user_api_client.py +++ b/app/notify_client/user_api_client.py @@ -157,6 +157,10 @@ class UserApiClient(NotifyAdminAPIClient): endpoint = "/user" return self.get(endpoint)["data"] + def get_all_users_detailed(self): + endpoint = "/user/report-all-users" + return self.get(endpoint)["data"] + @cache.delete("service-{service_id}") @cache.delete("service-{service_id}-template-folders") @cache.delete("user-{user_id}") diff --git a/app/templates/views/platform-admin/reports.html b/app/templates/views/platform-admin/reports.html index f22faa728..3b7b32d3a 100644 --- a/app/templates/views/platform-admin/reports.html +++ b/app/templates/views/platform-admin/reports.html @@ -31,4 +31,8 @@

Users Report

+

+ Download All Users +

+ {% endblock %} From 323ecf63f4cbcf2beafa35ce1130a7b1d0afb28a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 6 Aug 2024 12:27:42 -0700 Subject: [PATCH 02/13] remove unused import --- app/main/views/platform_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 07e973704..749b79c1a 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -3,7 +3,7 @@ import json from collections import OrderedDict from datetime import datetime -from flask import Response, abort, flash, render_template, request, send_file, url_for +from flask import Response, abort, flash, render_template, request, url_for from notifications_python_client.errors import HTTPError from app import ( From aed9778a6548919a3a58649f9324de62e7916bbc Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 6 Aug 2024 12:42:34 -0700 Subject: [PATCH 03/13] fix tests --- tests/app/test_navigation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/app/test_navigation.py b/tests/app/test_navigation.py index d346fc71a..aeee32164 100644 --- a/tests/app/test_navigation.py +++ b/tests/app/test_navigation.py @@ -68,6 +68,7 @@ EXCLUDED_ENDPOINTS = tuple( "delivery_status_callback", "design_content", "documentation", + "download_all_users", "download_notifications_csv", "download_organization_usage_report", "edit_and_format_messages", From 2497679046a11160727e552e1dcc710d80475df2 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 7 Aug 2024 07:21:53 -0700 Subject: [PATCH 04/13] switch commas to tabs --- app/main/views/platform_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 749b79c1a..67b57a0dd 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -75,7 +75,7 @@ def platform_admin(): def download_all_users(): # Create a CSV string from the user data - csv_data = "Name,Email Address,Mobile Number,Service\n" + csv_data = "Name\tEmail Address\tMobile Number\tService\n" users = user_api_client.get_all_users_detailed() if len(users) == 0: @@ -84,7 +84,7 @@ def download_all_users(): for user in users: if user["name"].startswith("e2e"): continue - csv_data += f"{user['name']},{user['email_address']},{user['mobile_number']},{user['service']}\n" + csv_data += f"{user['name']}\t{user['email_address']}\t{user['mobile_number']}\t{user['service']}\n" # Create a direct download response with the CSV data and appropriate headers response = Response(csv_data, content_type="text/csv") From 69e9c6d1cae51ef8d1297771ae6e73270700d948 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 10:47:18 -0700 Subject: [PATCH 05/13] code review changes --- app/main/views/platform_admin.py | 15 ++++++++++++--- poetry.lock | 6 +----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 67b57a0dd..0287dd6a9 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -1,3 +1,5 @@ +import csv +from io import StringIO import itertools import json from collections import OrderedDict @@ -80,12 +82,19 @@ def download_all_users(): if len(users) == 0: return "No data to download." - users = json.loads(users) + + output = StringIO() + writer = csv.DictWriter( + output, + fieldnames=["name", "email_address", "mobile_number", "service"], + delimiter="\t", + ) + writer.writeheader() for user in users: if user["name"].startswith("e2e"): continue - csv_data += f"{user['name']}\t{user['email_address']}\t{user['mobile_number']}\t{user['service']}\n" - + writer.writerow(user) + csv_data = output.getvalue() # Create a direct download response with the CSV data and appropriate headers response = Response(csv_data, content_type="text/csv") response.headers["Content-Disposition"] = "attachment; filename=users.csv" diff --git a/poetry.lock b/poetry.lock index 865e6da97..a636a17e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1325,13 +1325,9 @@ files = [ {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, @@ -1666,6 +1662,7 @@ files = [ {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"}, {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"}, {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"}, + {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"}, {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, ] @@ -2512,7 +2509,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From 1a75e56187430936da68cf7e8e70a7aebbfc53d7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 10:59:57 -0700 Subject: [PATCH 06/13] code review feedback --- app/main/views/platform_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 0287dd6a9..33fc614eb 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -1,9 +1,9 @@ import csv -from io import StringIO import itertools import json from collections import OrderedDict from datetime import datetime +from io import StringIO from flask import Response, abort, flash, render_template, request, url_for from notifications_python_client.errors import HTTPError From f3cab5a6ccbcfab574408c6146e4e2864508c708 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 11:22:08 -0700 Subject: [PATCH 07/13] revert Makefile changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff6396b49..90e8e76de 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ py-lint: ## Run python linting scanners and black poetry self add poetry-dotenv-plugin poetry run black . poetry run flake8 . - poetry run isort ./app ./tests + poetry run isort --check-only ./app ./tests .PHONY: avg-complexity avg-complexity: From ee1dcdeb0348c0cdb2737d44792169e4d6620ac7 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 13:02:51 -0700 Subject: [PATCH 08/13] strip commas from fields --- app/main/views/platform_admin.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 33fc614eb..e5b6b9760 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -77,24 +77,28 @@ def platform_admin(): def download_all_users(): # Create a CSV string from the user data - csv_data = "Name\tEmail Address\tMobile Number\tService\n" users = user_api_client.get_all_users_detailed() if len(users) == 0: return "No data to download." output = StringIO() + header = ["Name", "Email Address", "Phone Number", "Service"] + fieldnames = ["name", "email_address", "mobile_number", "service"] writer = csv.DictWriter( output, - fieldnames=["name", "email_address", "mobile_number", "service"], + fieldnames=fieldnames, delimiter="\t", ) - writer.writeheader() + # Write custom header + writer.writerow(dict(zip(fieldnames, header))) for user in users: - if user["name"].startswith("e2e"): + user_no_commas = {key: value.replace(",", "") for key, value in user.items()} + if user_no_commas["name"].startswith("e2e"): continue - writer.writerow(user) + writer.writerow(user_no_commas) csv_data = output.getvalue() + # Create a direct download response with the CSV data and appropriate headers response = Response(csv_data, content_type="text/csv") response.headers["Content-Disposition"] = "attachment; filename=users.csv" From 38a6c35afd8aab0daf5da0d2def7bc0d3672792c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 13:11:58 -0700 Subject: [PATCH 09/13] add test --- app/main/views/platform_admin.py | 2 +- tests/app/main/views/test_platform_admin.py | 33 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index e5b6b9760..07d3b46b5 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -100,7 +100,7 @@ def download_all_users(): csv_data = output.getvalue() # Create a direct download response with the CSV data and appropriate headers - response = Response(csv_data, content_type="text/csv") + response = Response(csv_data, content_type="text/csv; charset=utf-8") response.headers["Content-Disposition"] = "attachment; filename=users.csv" return response diff --git a/tests/app/main/views/test_platform_admin.py b/tests/app/main/views/test_platform_admin.py index e617b27f8..d4b237077 100644 --- a/tests/app/main/views/test_platform_admin.py +++ b/tests/app/main/views/test_platform_admin.py @@ -1254,3 +1254,36 @@ def test_get_daily_sms_provider_volumes_report_calls_api_and_download_data( + "80" + "\r\n" ) + + +def test_get_users_report(client_request, platform_admin_user, mocker): + mocker.patch( + "app.main.views.platform_admin.user_api_client.get_all_users_detailed", + return_value=[ + { + "name": "Johnny Sokko", + "email_address": "j_sokko@unicorn.gov", + "mobile_number": "15555555555", + "service": "Emperor, Guillotine, Service", + } + ], + ) + + client_request.login(platform_admin_user) + response = client_request.get_response( + "main.download_all_users", + _data={}, + _expected_status=200, + ) + + assert response.content_type == "text/csv; charset=utf-8" + assert "attachment" in response.headers["Content-Disposition"] + assert "filename" in response.headers["Content-Disposition"] + assert "users" in response.headers["Content-Disposition"] + + my_response = response.get_data(as_text=True) + + assert "Johnny Sokko" in my_response + assert "Emperor Guillotine Service" in my_response + assert "j_sokko@unicorn.gov" in my_response + assert "15555555555" in my_response From 193e1f7952089e90cc0328ceaba9d733907dfcec Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 8 Aug 2024 13:16:29 -0700 Subject: [PATCH 10/13] fix duplicate test name --- tests/app/main/views/test_platform_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/main/views/test_platform_admin.py b/tests/app/main/views/test_platform_admin.py index d4b237077..5124bd81d 100644 --- a/tests/app/main/views/test_platform_admin.py +++ b/tests/app/main/views/test_platform_admin.py @@ -1256,7 +1256,7 @@ def test_get_daily_sms_provider_volumes_report_calls_api_and_download_data( ) -def test_get_users_report(client_request, platform_admin_user, mocker): +def test_download_all_users(client_request, platform_admin_user, mocker): mocker.patch( "app.main.views.platform_admin.user_api_client.get_all_users_detailed", return_value=[ From bda957f76a21899a2ff8bed45e8aeac529bf9896 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 12 Aug 2024 07:27:32 -0700 Subject: [PATCH 11/13] change delimiter back to comma --- app/main/views/platform_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 07d3b46b5..f0b5983b5 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -88,7 +88,7 @@ def download_all_users(): writer = csv.DictWriter( output, fieldnames=fieldnames, - delimiter="\t", + delimiter=",", ) # Write custom header writer.writerow(dict(zip(fieldnames, header))) From b5d4e12466ae2d02ec0fd96004a35dc20d34f131 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 13 Aug 2024 11:37:08 -0700 Subject: [PATCH 12/13] fix formatting --- app/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index f0aa2e676..483d89ea0 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -175,7 +175,6 @@ def create_app(application): init_govuk_frontend(application) init_jinja(application) - for client in ( csrf, login_manager, From 3397b0c3804c393134e67a23de8a3b61dc3d1142 Mon Sep 17 00:00:00 2001 From: alexjanousekGSA Date: Tue, 13 Aug 2024 14:15:02 -0500 Subject: [PATCH 13/13] Added back part that got removed that allowed png images --- gulpfile.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gulpfile.js b/gulpfile.js index b441f4c2d..d4d6ac81f 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -99,10 +99,11 @@ const copyGtmHead = () => { // Task to copy images const copyImages = () => { - return src(paths.src + 'images/**/*') + return src(paths.src + 'images/**/*', { encoding: false }) .pipe(dest(paths.dist + 'images/')); }; + // Configure USWDS paths uswds.settings.version = 3; uswds.paths.dist.css = paths.dist + 'css';