Merge branch 'main' of https://github.com/GSA/notifications-admin into 1544-data-viz-total-message-allowance

app/main/views/send.py

@@ -3,7 +3,16 @@ import uuid
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
-from flask import abort, flash, redirect, render_template, request, session, url_for
+from flask import (
+    abort,
+    current_app,
+    flash,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user
 from markupsafe import Markup
 from notifications_python_client.errors import HTTPError
@@ -31,12 +40,18 @@ from app.s3_client.s3_csv_client import (
     s3upload,
     set_metadata_on_csv_upload,
 )
-from app.utils import PermanentRedirect, should_skip_template_page, unicode_truncate
+from app.utils import (
+    PermanentRedirect,
+    hilite,
+    should_skip_template_page,
+    unicode_truncate,
+)
 from app.utils.csv import Spreadsheet, get_errors_for_csv
 from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
 from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV, first_column_headings
 from notifications_utils.sanitise_text import SanitiseASCII
 
@@ -938,6 +953,9 @@ def send_notification(service_id, template_id):
             )
         )
 
+    current_app.logger.info(
+        hilite(scrub(f"Recipient for the one-off will be {recipient}"))
+    )
     keys = []
     values = []
     for k, v in session["placeholders"].items():
@@ -971,6 +989,19 @@ def send_notification(service_id, template_id):
         valid="True",
     )
 
+    # Here we are attempting to cleverly link the job id to the one-off recipient
+    # If we know the partial phone number of the recipient, we can search
+    # on that initially and find this, which will give us the job_id
+    # And once we know the job_id, we can search on that and it might tell us something
+    # about report generation.
+    current_app.logger.info(
+        hilite(
+            scrub(
+                f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}"
+            )
+        )
+    )
+
     session.pop("recipient")
     session.pop("placeholders")
 

app/templates/views/message-status.html

@@ -9,9 +9,13 @@
 
   <h1 class="font-body-2xl margin-bottom-3">Delivery status</h1>
 
-  <p>Notify’s real-time dashboard lets you check the status of any message.</p>
-  <p>For <a class="usa-link" href="{{ url_for('main.security') }}">security</a>, this information is only available for seven days after a message has been sent. You can download a report, including a list of sent messages, for your own records.</p>
-  <p>This page describes the statuses you'll see when you're signed in to Notify.</p>
+  <p>Notify starts sending your text messages immediately. Each message is sent to its cell phone carrier, which attempts
+  delivery to the recipient. This process is often almost instantaneous, but can sometimes take a while if a phone is
+  unavailable. The carrier will continue to try to deliver the message for up to 72 hours.</p>
+  <p>The Notify dashboard provides a high-level view of the number of messages Sent, Pending, Delivered, and Failed. The
+  dashboard data starts to update about five minutes after a message is sent and will continue to update for four hours.</p>
+  <p>Delivery statuses reflect delivery to a specific <em>cell phone number</em>, not to a <em>person</em>. No status can confirm that a
+  specific person actually received or read the message.</p>
 
   <!-- <p>If you’re using the Notify API, read our <a class="usa-link" href="{{ url_for('.documentation') }}">documentation</a> for a list of API statuses.<p>
 
@@ -45,7 +49,7 @@
   <p>Notify cannot tell you if your users open an email or click on the links in an email. We do not track open rates and click-throughs because there are privacy issues. Tracking emails without asking permission from users could breach General Data Protection Regulations (GDPR).</p> -->
 
   <h2 id="text-message-statuses" class="heading-medium">Text messages</h2>
-  <div class="bottom-gutter-3-2">
+  <div class="margin-bottom-8">
     {% call mapping_table(
       caption='Message statuses – text messages',
       field_headings=['Status', 'Description'],
@@ -53,12 +57,14 @@
       caption_visible=False
     ) %}
       {% for message_status, description in [
-        ('Total', 'The total number of messages that have been sent during the last seven days.'),
-        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72 hours. Notify is waiting for delivery information.'),
+        ('Scheduled', 'The total number of messages that have been scheduled to be sent at some future time.'),
+        ('Canceled', 'Messages that were created and scheduled, but canceled prior to Sending.'),
+        ('Total', 'The total number of messages that have been sent during the specified time.'),
+        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72
+        hours. “Pending” indicates that Notify is waiting for delivery information.'),
         ('Delivered', 'The message was successfully delivered. Notify cannot tell you if a user has opened or read a message.'),
-        ('Failed', 'The provider could not deliver the message. This can happen if the phone number was wrong or if the network operator
-        rejects the message. If you’re sure that these phone numbers are correct, you should <a class="usa-link" href="/support">contact us</a>. If not, you should remove them from your database. You’ll still be charged for text messages that
-        cannot be delivered.' | safe),
+        ('Failed', 'The message could not be delivered.'),
+        ('Process error / Delivery not attempted', 'If you receive a large number of process errors, please contact the Notify team.'),
         ] %}
         {% call row() %}
           {{ text_field(message_status) }}
@@ -67,5 +73,21 @@
       {% endfor %}
     {% endcall %}
   </div>
+  <h2 class="heading-medium">About carrier statuses</h2>
+  <p>Sometimes Notify receives more detailed information from the carriers on the status of messages, and these can be found
+  in the downloadable reports. Not all carriers provide the same level of detail regarding delivery and some delivery
+  statutes have a slight variation in word choice. Notify includes this information in the reports to provide you as much
+  detail as possible. Remember, for <a class="usa-link" href="/features/security">security</a> purposes, detailed information is only available for seven days after a
+  message has been sent.</p>
+
+  <h2 class="heading-medium">Opting out</h2>
+  <p>A text recipient can opt out of receiving text messages from your phone number at any time by responding “STOP” or
+  “QUIT” or <a class="usa-link" href="https://docs.aws.amazon.com/pinpoint/latest/userguide/channels-sms-limitations-opt-out.html#settings-sms-self-opt-out-terms">one of several other keywords</a>. If they opt out,</p>
+  <ul>
+    <li>They may receive a response from their carrier stating that they will no longer receive text messages from that number.</li>
+    <li>Any subsequent messages sent to that number will not be delivered (unless the recipient opts back in by texting “START”
+    or “OPT-IN”).</li>
+  </ul>
+  <p>Notify.gov does not yet have a way to pull opt-out status and make it available in the UI for agencies to download.</p>
 
 {% endblock %}

app/templates/views/trial-mode.html

@@ -21,7 +21,7 @@
     <li>A text message of 160-306 characters is two parts.</li>
   </ul>
   <p>For more information on how message parts are calculated, see
-  <a href="/using-notify/pricing">Pricing</a>.</p>
+  <a href="/using-notify/pricing">Tracking usage</a>.</p>
 
   <h2 class="font-body-lg">Before going Live</h2>
   <p>Before you request to make your service live so you can send messages to clients:</p>

app/utils/csv.py

@@ -1,10 +1,13 @@
 import datetime
 
 import pytz
+from flask import current_app, json
 from flask_login import current_user
 
 from app.models.spreadsheet import Spreadsheet
+from app.utils import hilite
 from app.utils.templates import get_sample_template
+from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV
 
 
@@ -71,7 +74,24 @@ def generate_notifications_csv(**kwargs):
 
     # This generates the "batch" csv report
     if kwargs.get("job_id"):
+        # The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug
+        # Some unit tests are mocking the kwargs and turning them into a function instead of dict,
+        # hence the try/except.
+        try:
+            current_app.logger.info(
+                hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}")
+            )
+        except TypeError:
+            pass
+
         original_file_contents = s3download(kwargs["service_id"], kwargs["job_id"])
+        # This will verify that the user actually did successfully upload a csv for a one-off.  Limit the size
+        # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows.
+        current_app.logger.info(
+            hilite(
+                f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}"
+            )
+        )
         original_upload = RecipientCSV(
             original_file_contents,
             template=get_sample_template(kwargs["template_type"]),