From a4f6dd7100adecd6d2c24135fe21d5fd1e135f51 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 22 Jan 2025 08:44:34 -0800 Subject: [PATCH 01/32] redis report --- app/main/views/platform_admin.py | 66 +++++++++++++++++++ app/main/views/sign_in.py | 11 ++-- .../views/platform-admin/reports.html | 3 + .../clients/redis/redis_client.py | 16 +++++ tests/app/main/views/test_tour.py | 4 +- 5 files changed, 92 insertions(+), 8 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 6a121ecc2..04ef87a55 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -116,6 +116,72 @@ def download_all_users(): return response +@main.route("/platform-admin/get-redis-report") +@user_is_platform_admin +def get_redis_report(): + + memory_info = redis_client.info("memory") + memory_used = memory_info.get("used_memory_human", "N/A") + max_memory = memory_info.get("maxmemory_human", "N/A") + if max_memory == "0B": + max_memory = "No set limit" + mem_fragmentation = memory_info.get("mem_fragmentation_ratio", "N/A") + frag_quality = "Swapping (bad)" + if mem_fragmentation >= 1.0: + frag_quality = "Healthy" + if mem_fragmentation > 1.5: + frag_quality = "Problematic" + if mem_fragmentation > 2.0: + frag_quality = "Severe fragmentation" + + frag_note = "" + if mem_fragmentation > 2.0: + frag_note = "Use MEMORY PURGE.\nReplace multiple small keys with hashes.\nAvoid long keys.\nSet max_memory." + elif mem_fragmentation < 1.0: + frag_note = "Allocate more RAM.\nSet max_memory." + + keys = redis_client.keys("*") + key_details = [] + + for key in keys: + key_type = redis_client.type(key).decode("utf-8") + ttl = redis_client.ttl(key) + ttl_str = "No Expiry" if ttl == -1 else f"{ttl} seconds" + key_details.append( + {"Key": key.decode("utf-8"), "Type": key_type, "TTL": ttl_str} + ) + output = StringIO() + writer = csv.writer( + output, + ) + writer.writerow(["Redis Report"]) + writer.writerow([]) + + writer.writerow(["Memory"]) + writer.writerow(["", "Metric", "Value"]) + writer.writerow(["", "Memory Used", memory_used]) + writer.writerow(["", "Max Memory", max_memory]) + writer.writerow(["", "Memory Fragmentation Ratio", mem_fragmentation]) + writer.writerow(["", "Memory Fragmentation Quality", frag_quality]) + writer.writerow(["", "Memory Fragmentation Note", frag_note]) + writer.writerow([]) + + writer.writerow(["Keys Overview"]) + writer.writerow(["", "TTL", "Type", "Key"]) + for key_detail in key_details: + writer.writerow( + ["", key_detail["TTL"], key_detail["Type"], key_detail["Key"][0:50]] + ) + + csv_data = output.getvalue() + + # Create a direct download response with the CSV data and appropriate headers + response = Response(csv_data, content_type="text/csv; charset=utf-8") + response.headers["Content-Disposition"] = "attachment; filename=redis.csv" + + return response + + def is_over_threshold(number, total, threshold): percentage = number / total * 100 if total else 0 return percentage > threshold diff --git a/app/main/views/sign_in.py b/app/main/views/sign_in.py index 004dce2ae..1cb163691 100644 --- a/app/main/views/sign_in.py +++ b/app/main/views/sign_in.py @@ -68,11 +68,12 @@ def _get_access_token(code): # pragma: no cover id_token = get_id_token(response_json) nonce = id_token["nonce"] nonce_key = f"login-nonce-{unquote(nonce)}" - stored_nonce = redis_client.get(nonce_key).decode("utf8") + if not os.getenv("NOTIFY_ENVIRONMENT") == "development": + stored_nonce = redis_client.get(nonce_key).decode("utf8") - if nonce != stored_nonce: - current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}") - abort(403) + if nonce != stored_nonce: + current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}") + abort(403) try: access_token = response_json["access_token"] @@ -112,7 +113,7 @@ def _do_login_dot_gov(): # $ pragma: no cover verify_key = f"login-verify_email-{unquote(state)}" verify_path = bool(redis_client.get(verify_key)) - if not verify_path: + if not verify_path and not os.getenv("NOTIFY_ENVIRONMENT") == "development": state_key = f"login-state-{unquote(state)}" stored_state = unquote(redis_client.get(state_key).decode("utf8")) if state != stored_state: diff --git a/app/templates/views/platform-admin/reports.html b/app/templates/views/platform-admin/reports.html index 3b7b32d3a..23dccc41c 100644 --- a/app/templates/views/platform-admin/reports.html +++ b/app/templates/views/platform-admin/reports.html @@ -34,5 +34,8 @@

Download All Users

+

+ Get Redis Report +

{% endblock %} diff --git a/notifications_utils/clients/redis/redis_client.py b/notifications_utils/clients/redis/redis_client.py index 1723dd2c1..469cd7c8f 100644 --- a/notifications_utils/clients/redis/redis_client.py +++ b/notifications_utils/clients/redis/redis_client.py @@ -149,6 +149,22 @@ class RedisClient: except Exception as e: self.__handle_exception(e, raise_exception, "incr", key) + def info(self, key): + if self.active: + return self.redis_store.info(key) + + def keys(self, pattern): + if self.active: + return self.redis_store.keys(pattern) + + def type(self, key): + if self.active: + return self.redis_store.type(key) + + def ttl(self, key): + if self.active: + return self.redis_store.ttl(key) + def get(self, key, raise_exception=False): key = prepare_value(key) if self.active: diff --git a/tests/app/main/views/test_tour.py b/tests/app/main/views/test_tour.py index 9a2b8f053..3207123a9 100644 --- a/tests/app/main/views/test_tour.py +++ b/tests/app/main/views/test_tour.py @@ -174,9 +174,7 @@ def test_should_show_empty_text_box( # data-module=autofocus is set on a containing element so it # shouldn’t also be set on the textbox itself assert "data-module" not in textbox - assert ( - normalize_spaces(page.select_one("label[for=phone-number]").text) == "one" - ) + assert normalize_spaces(page.select_one("label[for=phone-number]").text) == "one" def test_should_prefill_answers_for_get_tour_step( From cfb966ce5346e3081d3193e6cb50a8b87e0ce451 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 22 Jan 2025 13:34:34 -0800 Subject: [PATCH 02/32] redis report --- app/main/views/platform_admin.py | 5 ++--- tests/app/test_navigation.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 04ef87a55..783ce01b6 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -158,12 +158,11 @@ def get_redis_report(): writer.writerow([]) writer.writerow(["Memory"]) - writer.writerow(["", "Metric", "Value"]) writer.writerow(["", "Memory Used", memory_used]) writer.writerow(["", "Max Memory", max_memory]) writer.writerow(["", "Memory Fragmentation Ratio", mem_fragmentation]) - writer.writerow(["", "Memory Fragmentation Quality", frag_quality]) - writer.writerow(["", "Memory Fragmentation Note", frag_note]) + writer.writerow(["", "Memory Fragmentation Quality", frag_quality, frag_note]) + #writer.writerow(["", "Memory Fragmentation Note", frag_note]) writer.writerow([]) writer.writerow(["Keys Overview"]) diff --git a/tests/app/test_navigation.py b/tests/app/test_navigation.py index ed4d66182..ff6a9fb0e 100644 --- a/tests/app/test_navigation.py +++ b/tests/app/test_navigation.py @@ -107,6 +107,7 @@ EXCLUDED_ENDPOINTS = tuple( "get_volumes_by_service", "get_example_csv", "get_notifications_as_json", + "get_redis_report", "get_started", "get_started_old", "go_to_dashboard_after_tour", From 5d565ab88be0581ab6f148c73f946440599a5537 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 22 Jan 2025 13:45:42 -0800 Subject: [PATCH 03/32] fix black and isort forever --- .ds.baseline | 4 ++-- .github/workflows/checks.yml | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.ds.baseline b/.ds.baseline index 0ded707f2..71169119f 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -133,7 +133,7 @@ "filename": ".github/workflows/checks.yml", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 68, + "line_number": 70, "is_secret": false } ], @@ -684,5 +684,5 @@ } ] }, - "generated_at": "2025-01-16T16:38:48Z" + "generated_at": "2025-01-22T21:45:39Z" } diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c3ef5dcbb..4041a8923 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -38,10 +38,12 @@ jobs: output: report-markdown annotations: failed-tests prnumber: ${{ steps.findPr.outputs.number }} + - name: Run black + run: poetry run black . + - name: Check imports alphabetized + run: poetry run isort ./app ./tests - name: Run style checks run: poetry run flake8 . - - name: Check imports alphabetized - run: poetry run isort --check-only ./app ./tests - name: Check dead code run: make dead-code - name: Run js tests From 7776f9900a70555ac0901593eeb95b2f1a784402 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Mon, 27 Jan 2025 19:15:37 -0800 Subject: [PATCH 04/32] added id and required field --- .../uswds/_uswds-theme-custom-styles.scss | 4 ++++ .../components/components/input/template.njk | 6 ++++-- app/templates/components/textbox.html | 20 +++++++++---------- app/templates/views/edit-sms-template.html | 5 ++++- app/templates/views/send-test.html | 4 ++-- app/templates/views/templates/template.html | 4 ---- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/app/assets/sass/uswds/_uswds-theme-custom-styles.scss b/app/assets/sass/uswds/_uswds-theme-custom-styles.scss index 0bb1aab7e..ebce95061 100644 --- a/app/assets/sass/uswds/_uswds-theme-custom-styles.scss +++ b/app/assets/sass/uswds/_uswds-theme-custom-styles.scss @@ -1024,3 +1024,7 @@ nav.nav { font-size: units(3); font-weight: bold; } + +.form-control-error { + border: 4px solid #b10e1e +} diff --git a/app/templates/components/components/input/template.njk b/app/templates/components/components/input/template.njk index 7f5634651..4ea649dce 100644 --- a/app/templates/components/components/input/template.njk +++ b/app/templates/components/components/input/template.njk @@ -34,7 +34,7 @@ attributes: params.errorMessage.attributes, html: params.errorMessage.html, text: params.errorMessage.text, - visuallyHiddenText: params.errorMessage.visuallyHiddenText + visuallyHiddenText: params.errorMessage.visuallyHiddenText, }) | indent(2) | trim }} {% endif %} + {%- for attribute, value in params.attributes %} {{ attribute }}="{{ value }}"{% endfor -%} + {%- if params.required %} required{% endif %} + /> diff --git a/app/templates/components/textbox.html b/app/templates/components/textbox.html index 3e479cbce..fa92d0cf8 100644 --- a/app/templates/components/textbox.html +++ b/app/templates/components/textbox.html @@ -16,19 +16,9 @@ placeholder='' ) %}
- {% if field.errors %} - - {% endif %}
{% endif %} + {% if field.errors %} + + Error: + {% if not safe_error_message %}{{ field.errors[0] }}{% else %}{{ field.errors[0]|safe }}{% endif %} + + {% endif %} {% if highlight_placeholders or autosize %} @@ -59,6 +55,8 @@ data_highlight_placeholders='true' if highlight_placeholders else 'false', rows=rows|string, placeholder=placeholder, + aria_describedby=field.name+"-error", + required='required' if required else None, **kwargs ) }} {% if suffix %} diff --git a/app/templates/views/edit-sms-template.html b/app/templates/views/edit-sms-template.html index 97eac73dc..8ef41bdb2 100644 --- a/app/templates/views/edit-sms-template.html +++ b/app/templates/views/edit-sms-template.html @@ -32,6 +32,8 @@
{{ form.name(param_extensions={ "extra_form_group_classes": "margin-bottom-2", + "id": "name", + "required": True, "hint": {"text": "Your recipients will not see this"} }) }} {{ textbox( @@ -41,7 +43,8 @@ hint=content_hint, rows=5, extra_form_group_classes='margin-bottom-1', - placeholder='Edit me! Check out the Personalization section below for details on cool ((stuff)) you can do with your messages!' + placeholder='Edit me! Check out the Personalization section below for details on cool ((stuff)) you can do with your messages!', + required=True ) }} {% if current_user.platform_admin %} {{ form.process_type }} diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index 720849ae6..be3b32a9a 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -37,8 +37,8 @@ data_kwargs={'force-focus': True} ) %}
-