Merge branch 'main' of https://github.com/GSA/notifications-admin into 1484-dashboard-visualizations

app/main/views/send.py

@@ -3,7 +3,16 @@ import uuid
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
-from flask import abort, flash, redirect, render_template, request, session, url_for
+from flask import (
+    abort,
+    current_app,
+    flash,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user
 from markupsafe import Markup
 from notifications_python_client.errors import HTTPError
@@ -31,12 +40,18 @@ from app.s3_client.s3_csv_client import (
     s3upload,
     set_metadata_on_csv_upload,
 )
-from app.utils import PermanentRedirect, should_skip_template_page, unicode_truncate
+from app.utils import (
+    PermanentRedirect,
+    hilite,
+    should_skip_template_page,
+    unicode_truncate,
+)
 from app.utils.csv import Spreadsheet, get_errors_for_csv
 from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
 from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV, first_column_headings
 from notifications_utils.sanitise_text import SanitiseASCII
 
@@ -938,6 +953,9 @@ def send_notification(service_id, template_id):
             )
         )
 
+    current_app.logger.info(
+        hilite(scrub(f"Recipient for the one-off will be {recipient}"))
+    )
     keys = []
     values = []
     for k, v in session["placeholders"].items():
@@ -971,6 +989,19 @@ def send_notification(service_id, template_id):
         valid="True",
     )
 
+    # Here we are attempting to cleverly link the job id to the one-off recipient
+    # If we know the partial phone number of the recipient, we can search
+    # on that initially and find this, which will give us the job_id
+    # And once we know the job_id, we can search on that and it might tell us something
+    # about report generation.
+    current_app.logger.info(
+        hilite(
+            scrub(
+                f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}"
+            )
+        )
+    )
+
     session.pop("recipient")
     session.pop("placeholders")
 

app/templates/views/message-status.html

@@ -9,9 +9,13 @@
 
   <h1 class="font-body-2xl margin-bottom-3">Delivery status</h1>
 
-  <p>Notify’s real-time dashboard lets you check the status of any message.</p>
-  <p>For <a class="usa-link" href="{{ url_for('main.security') }}">security</a>, this information is only available for seven days after a message has been sent. You can download a report, including a list of sent messages, for your own records.</p>
-  <p>This page describes the statuses you'll see when you're signed in to Notify.</p>
+  <p>Notify starts sending your text messages immediately. Each message is sent to its cell phone carrier, which attempts
+  delivery to the recipient. This process is often almost instantaneous, but can sometimes take a while if a phone is
+  unavailable. The carrier will continue to try to deliver the message for up to 72 hours.</p>
+  <p>The Notify dashboard provides a high-level view of the number of messages Sent, Pending, Delivered, and Failed. The
+  dashboard data starts to update about five minutes after a message is sent and will continue to update for four hours.</p>
+  <p>Delivery statuses reflect delivery to a specific <em>cell phone number</em>, not to a <em>person</em>. No status can confirm that a
+  specific person actually received or read the message.</p>
 
   <!-- <p>If you’re using the Notify API, read our <a class="usa-link" href="{{ url_for('.documentation') }}">documentation</a> for a list of API statuses.<p>
 
@@ -45,7 +49,7 @@
   <p>Notify cannot tell you if your users open an email or click on the links in an email. We do not track open rates and click-throughs because there are privacy issues. Tracking emails without asking permission from users could breach General Data Protection Regulations (GDPR).</p> -->
 
   <h2 id="text-message-statuses" class="heading-medium">Text messages</h2>
-  <div class="bottom-gutter-3-2">
+  <div class="margin-bottom-8">
     {% call mapping_table(
       caption='Message statuses – text messages',
       field_headings=['Status', 'Description'],
@@ -53,12 +57,14 @@
       caption_visible=False
     ) %}
       {% for message_status, description in [
-        ('Total', 'The total number of messages that have been sent during the last seven days.'),
-        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72 hours. Notify is waiting for delivery information.'),
+        ('Scheduled', 'The total number of messages that have been scheduled to be sent at some future time.'),
+        ('Canceled', 'Messages that were created and scheduled, but canceled prior to Sending.'),
+        ('Total', 'The total number of messages that have been sent during the specified time.'),
+        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72
+        hours. “Pending” indicates that Notify is waiting for delivery information.'),
         ('Delivered', 'The message was successfully delivered. Notify cannot tell you if a user has opened or read a message.'),
-        ('Failed', 'The provider could not deliver the message. This can happen if the phone number was wrong or if the network operator
-        rejects the message. If you’re sure that these phone numbers are correct, you should <a class="usa-link" href="/support">contact us</a>. If not, you should remove them from your database. You’ll still be charged for text messages that
-        cannot be delivered.' | safe),
+        ('Failed', 'The message could not be delivered.'),
+        ('Process error / Delivery not attempted', 'If you receive a large number of process errors, please contact the Notify team.'),
         ] %}
         {% call row() %}
           {{ text_field(message_status) }}
@@ -67,5 +73,21 @@
       {% endfor %}
     {% endcall %}
   </div>
+  <h2 class="heading-medium">About carrier statuses</h2>
+  <p>Sometimes Notify receives more detailed information from the carriers on the status of messages, and these can be found
+  in the downloadable reports. Not all carriers provide the same level of detail regarding delivery and some delivery
+  statutes have a slight variation in word choice. Notify includes this information in the reports to provide you as much
+  detail as possible. Remember, for <a class="usa-link" href="/features/security">security</a> purposes, detailed information is only available for seven days after a
+  message has been sent.</p>
+
+  <h2 class="heading-medium">Opting out</h2>
+  <p>A text recipient can opt out of receiving text messages from your phone number at any time by responding “STOP” or
+  “QUIT” or <a class="usa-link" href="https://docs.aws.amazon.com/pinpoint/latest/userguide/channels-sms-limitations-opt-out.html#settings-sms-self-opt-out-terms">one of several other keywords</a>. If they opt out,</p>
+  <ul>
+    <li>They may receive a response from their carrier stating that they will no longer receive text messages from that number.</li>
+    <li>Any subsequent messages sent to that number will not be delivered (unless the recipient opts back in by texting “START”
+    or “OPT-IN”).</li>
+  </ul>
+  <p>Notify.gov does not yet have a way to pull opt-out status and make it available in the UI for agencies to download.</p>
 
 {% endblock %}

app/templates/views/trial-mode.html

@@ -21,7 +21,7 @@
     <li>A text message of 160-306 characters is two parts.</li>
   </ul>
   <p>For more information on how message parts are calculated, see
-  <a href="/using-notify/pricing">Pricing</a>.</p>
+  <a href="/using-notify/pricing">Tracking usage</a>.</p>
 
   <h2 class="font-body-lg">Before going Live</h2>
   <p>Before you request to make your service live so you can send messages to clients:</p>

app/utils/csv.py

@@ -1,10 +1,13 @@
 import datetime
 
 import pytz
+from flask import current_app, json
 from flask_login import current_user
 
 from app.models.spreadsheet import Spreadsheet
+from app.utils import hilite
 from app.utils.templates import get_sample_template
+from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV
 
 
@@ -71,7 +74,24 @@ def generate_notifications_csv(**kwargs):
 
     # This generates the "batch" csv report
     if kwargs.get("job_id"):
+        # The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug
+        # Some unit tests are mocking the kwargs and turning them into a function instead of dict,
+        # hence the try/except.
+        try:
+            current_app.logger.info(
+                hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}")
+            )
+        except TypeError:
+            pass
+
         original_file_contents = s3download(kwargs["service_id"], kwargs["job_id"])
+        # This will verify that the user actually did successfully upload a csv for a one-off.  Limit the size
+        # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows.
+        current_app.logger.info(
+            hilite(
+                f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}"
+            )
+        )
         original_upload = RecipientCSV(
             original_file_contents,
             template=get_sample_template(kwargs["template_type"]),

notifications_utils/logging.py

@@ -1,5 +1,6 @@
 import logging
 import logging.handlers
+import re
 import sys
 from itertools import product
 
@@ -131,3 +132,15 @@ class JSONFormatter(BaseJSONFormatter):
         except (KeyError, IndexError) as e:
             logger.exception("failed to format log message: {} not found".format(e))
         return log_record
+
+
+def scrub(msg):
+    # Eventually we want to scrub all messages in all logs for phone numbers
+    # and email addresses, masking them.  Ultimately this will probably get
+    # refactored into a 'SafeLogger' subclass or something, but let's start here
+    # with phones.
+    phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
+    phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
+    for phone in phones:
+        msg = msg.replace(phone, f"1XXXXX{phone[-5:]}")
+    return msg

poetry.lock

@@ -1641,6 +1641,7 @@ files = [
     {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
     {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
     {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
+    {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
     {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
 ]
 

tests/app/main/views/test_index.py

@@ -179,26 +179,6 @@ def test_old_static_pages_redirect(client_request, view, expected_view):
     )
 
 
-def test_message_status_page_contains_message_status_ids(client_request):
-    # The 'email-statuses' and 'sms-statuses' id are linked to when we display a message status,
-    # so this test ensures we don't accidentally remove them
-    page = client_request.get("main.message_status")
-
-    # email-statuses is commented out in view
-    # assert page.find(id='email-statuses')
-    assert page.find(id="text-message-statuses")
-
-
-def test_message_status_page_contains_link_to_support(client_request):
-    page = client_request.get("main.message_status")
-    sms_status_table = page.find(id="text-message-statuses").findNext("tbody")
-
-    temp_fail_details_cell = sms_status_table.select_one(
-        "tr:nth-child(4) > td:nth-child(2)"
-    )
-    assert temp_fail_details_cell.find("a").attrs["href"] == url_for("main.support")
-
-
 def test_old_using_notify_page(client_request):
     client_request.get("main.using_notify", _expected_status=410)
 

tests/notifications_utils/test_logging.py

@@ -49,3 +49,13 @@ def test_base_json_formatter_contains_service_id():
         == "message to log"
     )
     assert service_id_filter.filter(record).service_id == "no-service-id"
+
+
+def test_scrub():
+    result = logging.scrub(
+        "This is a message with 17775554324, and also 18884449323 and also 17775554324"
+    )
+    assert (
+        result
+        == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324"
+    )