Merge branch 'main' of https://github.com/GSA/notifications-admin into 1484-dashboard-visualizations

.ds.baseline

@@ -169,7 +169,7 @@
         "filename": "app/config.py",
         "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
         "is_verified": false,
-        "line_number": 111,
+        "line_number": 117,
         "is_secret": false
       }
     ],

.github/workflows/deploy-demo.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/demo
+            terraform/demo/**
-            terraform/shared
+            terraform/shared/**
             .github/workflows/deploy-demo.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-demo.*.acl

.github/workflows/deploy-prod.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/production
+            terraform/production/**
-            terraform/shared
+            terraform/shared/**
             .github/workflows/deploy-prod.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-production.*.acl

.github/workflows/deploy.yml

@@ -17,96 +17,96 @@ jobs:
 
     environment: staging
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v4
-        with:
+      with:
-          fetch-depth: 2
+        fetch-depth: 2
 
-      - name: Check for changes to Terraform
+    - name: Check for changes to Terraform
-        id: changed-terraform-files
+      id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+      uses: tj-actions/changed-files@v44
-        with:
+      with:
-          files: |
+        files: |
-            terraform/staging
+          terraform/staging/**
-            terraform/shared
+          terraform/shared/**
-            .github/workflows/deploy.yml
+          .github/workflows/deploy.yml
-      - name: Terraform init
+    - name: Terraform init
-        if: steps.changed-terraform-files.outputs.any_changed == 'true'
+      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-        working-directory: terraform/staging
+      working-directory: terraform/staging
-        env:
+      env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
+      run: terraform init
-      - name: Terraform apply
+    - name: Terraform apply
-        if: steps.changed-terraform-files.outputs.any_changed == 'true'
+      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-        working-directory: terraform/staging
+      working-directory: terraform/staging
-        env:
+      env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
+        TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
+        TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform apply -auto-approve -input=false
+      run: terraform apply -auto-approve -input=false
 
-      - uses: ./.github/actions/setup-project
+    - uses: ./.github/actions/setup-project
 
-      - name: Create requirements.txt
+    - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
 
 
-      - name: Deploy to cloud.gov
+    - name: Deploy to cloud.gov
-        uses: 18f/cg-deploy-action@main
+      uses: 18f/cg-deploy-action@main
-        env:
+      env:
-          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
+        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+        SECRET_KEY: ${{ secrets.SECRET_KEY }}
-          ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
+        ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
-          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-          NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
+        NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
-          COMMIT_HASH: ${{ github.sha }}
+        COMMIT_HASH: ${{ github.sha }}
-          LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
+        LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
-          LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
+        LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
-          LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
+        LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
-          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
+        LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
-          LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
+        LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
+        LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
-          LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
+        LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
+        LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
-        with:
+      with:
-          cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
+        cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-          cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
+        cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio
+        cf_org: gsa-tts-benefits-studio
-          cf_space: notify-staging
+        cf_space: notify-staging
-          push_arguments: >-
+        push_arguments: >-
-            --vars-file deploy-config/staging.yml
+          --vars-file deploy-config/staging.yml
-            --var DANGEROUS_SALT="$DANGEROUS_SALT"
+          --var DANGEROUS_SALT="$DANGEROUS_SALT"
-            --var SECRET_KEY="$SECRET_KEY"
+          --var SECRET_KEY="$SECRET_KEY"
-            --var ADMIN_CLIENT_USERNAME="notify-admin"
+          --var ADMIN_CLIENT_USERNAME="notify-admin"
-            --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
+          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-            --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-            --var NR_BROWSER_KEY="$NR_BROWSER_KEY"
+          --var NR_BROWSER_KEY="$NR_BROWSER_KEY"
-            --var COMMIT_HASH="$COMMIT_HASH"
+          --var COMMIT_HASH="$COMMIT_HASH"
-            --var LOGIN_PEM="$LOGIN_PEM"
+          --var LOGIN_PEM="$LOGIN_PEM"
-            --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
+          --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
-            --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
+          --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
-            --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
+          --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
-            --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
+          --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
-            --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
+          --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
-            --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
+          --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
-            --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
+          --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
 
 
-      - name: Check for changes to egress config
+    - name: Check for changes to egress config
-        id: changed-egress-config
+      id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+      uses: tj-actions/changed-files@v44
-        with:
+      with:
-          files: |
+        files: |
-            deploy-config/egress_proxy/notify-admin-staging.*.acl
+          deploy-config/egress_proxy/notify-admin-staging.*.acl
-            .github/actions/deploy-proxy/action.yml
+          .github/actions/deploy-proxy/action.yml
-            .github/workflows/deploy.yml
+          .github/workflows/deploy.yml
-      - name: Deploy egress proxy
+    - name: Deploy egress proxy
-        if: steps.changed-egress-config.outputs.any_changed == 'true'
+      if: steps.changed-egress-config.outputs.any_changed == 'true'
-        uses: ./.github/actions/deploy-proxy
+      uses: ./.github/actions/deploy-proxy
-        with:
+      with:
-          cf_space: notify-staging
+        cf_space: notify-staging
-          app: notify-admin-staging
+        app: notify-admin-staging
 
   bail:
     runs-on: ubuntu-latest

README.md

@@ -40,7 +40,7 @@ You will need the following items:
 [Follow the instructions here to set up the Notify.gov API.](https://github.com/GSA/notifications-api#before-you-start)
 
 The Notify.gov API is required in order for the Notify.gov Admin UI to run, and
-it will also take care of many of the steps that are listed here.  The sections
+it will also take care of many of the steps that are listed here. The sections
 that are a repeat from the API setup are flagged with an **[API Step]** label
 in front of them.
 
@@ -83,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these
 locations:
 
 For BASH shells:
+
 - `~/.bashrc`
 - `~/.bash_profile`
 - `~/.profile`
 
 For ZSH shells:
+
 - `~/.zshrc`
 - `~/.zprofile`
 
@@ -97,7 +99,7 @@ environments.
 Which file you need to modify depends on whether or not you are running an
 interactive shell or a login shell
 (see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell)
-for an explanation of the differences).  If you're still not sure, please ask
+for an explanation of the differences). If you're still not sure, please ask
 the team for help!
 
 Once you determine which file you'll need to modify, add these lines before any
@@ -158,7 +160,7 @@ _NOTE: This project currently uses the latest `1.4.x release of Terraform._
 #### [API Step] Python Installation
 
 Now we're going to install a tool to help us manage Python versions and
-virtual environments on our system.  First, we'll install
+virtual environments on our system. First, we'll install
 [pyenv](https://github.com/pyenv/pyenv) and one of its plugins,
 [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew:
 
@@ -285,7 +287,7 @@ we'll use `3.12` in our example here since we recently upgraded to this version:
 pyenv install 3.12
 ```
 
-Next, delete the virtual environment you previously had set up.  If you followed
+Next, delete the virtual environment you previously had set up. If you followed
 the instructions above with the first-time set up, you can do this with `pyenv`:
 
 ```sh
@@ -306,6 +308,20 @@ you'll be set with an upgraded version of Python.
 
 _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
 
+#### Updating the .env file for Login.gov
+
+To configure the application for Login.gov, you will need to update the following environment variables in the .env file:
+
+```
+COMMIT_HASH=”--------”
+```
+
+Reach out to someone on the team to get the most recent Login.gov key.
+
+```
+LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE"
+```
+
 #### Updating the .env file for E2E tests
 
 With the newly created `.env` file in place, you'll need to make one more
@@ -353,7 +369,7 @@ API is running as well!
 ## Creating a 'First User' in the database
 
 After you have completed all setup steps, you will be unable to log in, because there
-will not be a user in the database to link to the login.gov account you are using.  So
+will not be a user in the database to link to the login.gov account you are using. So
 you will need to create that user in your database using the 'create-test-user' command.
 
 Open two terminals pointing to the api project and then run these commands in the
@@ -372,8 +388,6 @@ is the same one you are using in login.gov and make sure your phone number is in
 If for any reason in the course of development it is necessary for your to delete your db
 via the `dropdb` command, you will need to repeat these steps when you recreate your db.
 
-
-
 ## Git Hooks
 
 We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to

app/config.py

@@ -53,7 +53,13 @@ class Config(object):
     PERMANENT_SESSION_LIFETIME = 1800  # 30 Minutes
     SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60  # 1 year
     REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45
-    ACTIVITY_STATS_LIMIT_DAYS = 7
+    ACTIVITY_STATS_LIMIT_DAYS = {
+        "today": 0,
+        "one_day": 1,
+        "three_day": 3,
+        "five_day": 5,
+        "seven_day": 7,
+    }
     SESSION_COOKIE_HTTPONLY = True
     SESSION_COOKIE_NAME = "notify_admin_session"
     SESSION_COOKIE_SECURE = True

app/main/views/jobs.py

@@ -143,11 +143,40 @@ def view_notifications(service_id, message_type=None):
             True: ["reference"],
             False: [],
         }.get(bool(current_service.api_keys)),
-        download_link=url_for(
+        download_link_one_day=url_for(
             ".download_notifications_csv",
             service_id=current_service.id,
             message_type=message_type,
             status=request.args.get("status"),
+            number_of_days="one_day",
+        ),
+        download_link_today=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="today",
+        ),
+        download_link_three_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="three_day",
+        ),
+        download_link_five_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="five_day",
+        ),
+        download_link_seven_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="seven_day",
         ),
     )
 
@@ -183,10 +212,9 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     filter_args["status"] = set_status_filters(filter_args)
     service_data_retention_days = None
     search_term = request.form.get("to", "")
-
     if message_type is not None:
         service_data_retention_days = current_service.get_days_of_retention(
-            message_type
+            message_type, number_of_days="seven_day"
         )
 
     if request.path.endswith("csv") and current_user.has_permissions("view_activity"):
@@ -212,7 +240,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     )
     url_args = {"message_type": message_type, "status": request.args.get("status")}
     prev_page = None
-
     if "links" in notifications and notifications["links"].get("prev", None):
         prev_page = generate_previous_dict(
             "main.view_notifications", service_id, page, url_args=url_args
@@ -233,7 +260,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
         )
     else:
         download_link = None
-
     return {
         "service_data_retention_days": service_data_retention_days,
         "counts": render_template(
@@ -362,6 +388,7 @@ def get_job_partials(job):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
     notifications = job.get_notifications(status=filter_args["status"])
+    number_of_days = "seven_day"
     counts = render_template(
         "partials/count.html",
         counts=_get_job_counts(job),
@@ -371,7 +398,7 @@ def get_job_partials(job):
         ),
     )
     service_data_retention_days = current_service.get_days_of_retention(
-        job.template_type
+        job.template_type, number_of_days
     )
 
     if request.referrer is not None:

app/main/views/notifications.py

@@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification):
 def download_notifications_csv(service_id):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
-
+    number_of_days = request.args["number_of_days"]
     service_data_retention_days = current_service.get_days_of_retention(
-        filter_args.get("message_type")[0]
+        filter_args.get("message_type")[0], number_of_days
     )
     file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
     file_time = f"{file_time} {get_user_preferred_timezone()}"

app/main/views/send.py

@@ -51,7 +51,6 @@ from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
 from notifications_utils.insensitive_dict import InsensitiveDict
-from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV, first_column_headings
 from notifications_utils.sanitise_text import SanitiseASCII
 
@@ -953,9 +952,6 @@ def send_notification(service_id, template_id):
             )
         )
 
-    current_app.logger.info(
-        hilite(scrub(f"Recipient for the one-off will be {recipient}"))
-    )
     keys = []
     values = []
     for k, v in session["placeholders"].items():
@@ -971,6 +967,12 @@ def send_notification(service_id, template_id):
     )
     my_data = {"filename": filename, "template_id": template_id, "data": data}
     upload_id = s3upload(service_id, my_data)
+
+    # To debug messages that the user reports have not been sent, we log
+    # the csv filename and the job id.  The user will give us the file name,
+    # so we can search on that to obtain the job id, which we can use elsewhere
+    # on the API side to find out what happens to the message.
+    current_app.logger.info(hilite(f"One-off file: {filename} job_id: {upload_id}"))
     form = CsvUploadForm()
     form.file.data = my_data
     form.file.name = filename
@@ -989,19 +991,6 @@ def send_notification(service_id, template_id):
         valid="True",
     )
 
-    # Here we are attempting to cleverly link the job id to the one-off recipient
-    # If we know the partial phone number of the recipient, we can search
-    # on that initially and find this, which will give us the job_id
-    # And once we know the job_id, we can search on that and it might tell us something
-    # about report generation.
-    current_app.logger.info(
-        hilite(
-            scrub(
-                f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}"
-            )
-        )
-    )
-
     session.pop("recipient")
     session.pop("placeholders")
 
@@ -1033,7 +1022,12 @@ def send_notification(service_id, template_id):
                 job_id=upload_id,
             )
         )
-
+    total = notifications["total"]
+    current_app.logger.info(
+        hilite(
+            f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}"
+        )
+    )
     return redirect(
         url_for(
             ".view_job",

app/models/service.py

@@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin):
     def get_data_retention_item(self, id):
         return next((dr for dr in self.data_retention if dr["id"] == id), None)
 
-    def get_days_of_retention(self, notification_type):
+    def get_days_of_retention(self, notification_type, number_of_days):
         return next(
             (
                 dr
@@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin):
                 if dr["notification_type"] == notification_type
             ),
             {},
-        ).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"])
+        ).get(
+            "days_of_retention",
+            current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days),
+        )
 
     @cached_property
     def organization(self):

app/templates/views/notifications.html

@@ -64,10 +64,22 @@
 
   {% if current_user.has_permissions('view_activity') %}
     <p class="font-body-sm">
-      <a href="{{ download_link }}" download="download" class="usa-link">Download this report (<abbr title="Comma separated values">CSV</abbr>)</a>
+      <a href="{{ download_link_seven_day }}" download="download" class="usa-link">Download all data last 7 days (<abbr title="Comma separated values">CSV</abbr>)</a>
       &emsp;
       Data available for {{ partials.service_data_retention_days }} days
     </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_five_day }}" download="download" class="usa-link">Download all data last 5 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_three_day }}" download="download" class="usa-link">Download all data last 3 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_today }}" download="download" class="usa-link">Download all data today (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
   {% endif %}
 
   {{ ajax_block(

app/utils/csv.py

@@ -7,7 +7,6 @@ from flask_login import current_user
 from app.models.spreadsheet import Spreadsheet
 from app.utils import hilite
 from app.utils.templates import get_sample_template
-from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV
 
 
@@ -74,12 +73,11 @@ def generate_notifications_csv(**kwargs):
 
     # This generates the "batch" csv report
     if kwargs.get("job_id"):
-        # The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug
         # Some unit tests are mocking the kwargs and turning them into a function instead of dict,
         # hence the try/except.
         try:
             current_app.logger.info(
-                hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}")
+                hilite(f"Setting up report with kwargs {json.dumps(kwargs)}")
             )
         except TypeError:
             pass
@@ -89,7 +87,7 @@ def generate_notifications_csv(**kwargs):
         # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows.
         current_app.logger.info(
             hilite(
-                f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}"
+                f"Original csv for job_id {kwargs['job_id']}: {original_file_contents[0:999]}"
             )
         )
         original_upload = RecipientCSV(

deploy-config/production.yml

@@ -1,6 +1,6 @@
 env: production
 instances: 2
-memory: 1G
+memory: 2G
 public_admin_route: beta.notify.gov
 cloud_dot_gov_route: notify.app.cloud.gov
 redis_enabled: 1

gunicorn_config.py

@@ -5,7 +5,10 @@ import multiprocessing
 import gunicorn
 
 # Let gunicorn figure out the right number of workers
-workers = multiprocessing.cpu_count() * 2 + 1
+# The recommended formula is cpu_count() * 2 + 1
+# but we have an unusual configuration with a lot of cpus and not much memory
+# so adjust it.
+workers = multiprocessing.cpu_count()
 worker_class = "eventlet"
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))
 disable_redirect_access_to_syslog = True
@@ -16,23 +19,3 @@ def worker_abort(worker):
     worker.log.info("worker received ABORT")
     for stack in sys._current_frames().values():
         worker.log.error("".join(traceback.format_stack(stack)))
-
-
-# This issue is fixed in the 22.0.0 release, which we are using
-# See github issue for details
-# def fix_ssl_monkeypatching():
-#     """
-#     eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently
-#     a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However
-#     eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not
-#     ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type
-#     :scream:
-#     https://github.com/eventlet/eventlet/issues/692
-#     """
-#     # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again.
-#     # eventlet.monkey_patch can be called multiple times without issue
-#     eventlet.monkey_patch()
-#     eventlet.green.ssl.timeout_exc = socket.timeout
-
-
-# fix_ssl_monkeypatching()

manifest.yml

@@ -12,7 +12,7 @@ applications:
       - route: ((cloud_dot_gov_route))
 
     services:
-      - notify-admin-redis-((env))
+      - notify-admin-redis-v70-((env))
       - notify-api-csv-upload-bucket-((env))
       - notify-admin-logo-upload-bucket-((env))
 

notifications_utils/logging.py

@@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter):
     handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"]))
     handler.addFilter(RequestIdFilter())
     handler.addFilter(ServiceIdFilter())
+    handler.addFilter(PIIFilter())
 
     return handler
 
@@ -134,13 +135,30 @@ class JSONFormatter(BaseJSONFormatter):
         return log_record
 
 
-def scrub(msg):
+class PIIFilter(logging.Filter):
-    # Eventually we want to scrub all messages in all logs for phone numbers
+    def scrub(self, msg):
-    # and email addresses, masking them.  Ultimately this will probably get
+        # Eventually we want to scrub all messages in all logs for phone numbers
-    # refactored into a 'SafeLogger' subclass or something, but let's start here
+        # and email addresses, masking them.  Ultimately this will probably get
-    # with phones.
+        # refactored into a 'SafeLogger' subclass or something, but let's start here
-    phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
+        # with phones.
-    phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
+
-    for phone in phones:
+        # Sometimes just an exception object is passed in for the message, skip those.
-        msg = msg.replace(phone, f"1XXXXX{phone[-5:]}")
+        if not isinstance(msg, str):
-    return msg
+            return msg
+        phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
+        phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
+        for phone in phones:
+            msg = msg.replace(phone, "1XXXXXXXXXX")
+
+        emails = re.findall(
+            r"[\w\.-]+@[\w\.-]+", msg
+        )  # ['alice@google.com', 'bob@abc.com']
+        for email in emails:
+            # do something with each found email string
+            masked_email = "XXXXX@XXXXXXX"
+            msg = msg.replace(email, masked_email)
+        return msg
+
+    def filter(self, record):
+        record.msg = self.scrub(record.msg)
+        return record

terraform/bootstrap/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/demo/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = false
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/demo/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/development/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/production/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = false
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-3node-large"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-3node-large"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/production/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/sandbox/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = true
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/sandbox/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/shared/container_networking/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/staging/main.tf

@@ -6,7 +6,15 @@ locals {
   recursive_delete = true
 }
 
-module "redis" {
+resource "null_resource" "prevent_destroy" {
+
+  lifecycle {
+    prevent_destroy = false # destroying staging is allowed
+  }
+}
+
+
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +24,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/staging/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

tests/app/main/views/test_activity.py

@@ -228,12 +228,18 @@ def test_can_show_notifications_if_data_retention_not_available(
                 url_for,
                 ".download_notifications_csv",
                 message_type=None,
+                number_of_days="seven_day",
             ),
         ),
         (
             create_active_user_with_permissions(),
             {"status": "failed"},
-            partial(url_for, ".download_notifications_csv", status="failed"),
+            partial(
+                url_for,
+                ".download_notifications_csv",
+                status="failed",
+                number_of_days="seven_day",
+            ),
         ),
         (
             create_active_user_with_permissions(),
@@ -242,15 +248,13 @@ def test_can_show_notifications_if_data_retention_not_available(
                 url_for,
                 ".download_notifications_csv",
                 message_type="sms",
+                number_of_days="seven_day",
             ),
         ),
         (
             create_active_user_view_permissions(),
             {},
-            partial(
+            partial(url_for, ".download_notifications_csv", number_of_days="seven_day"),
-                url_for,
-                ".download_notifications_csv",
-            ),
         ),
         (
             create_active_caseworking_user(),

tests/notifications_utils/test_logging.py

@@ -51,11 +51,16 @@ def test_base_json_formatter_contains_service_id():
     assert service_id_filter.filter(record).service_id == "no-service-id"
 
 
-def test_scrub():
+def test_pii_filter():
-    result = logging.scrub(
+    record = builtin_logging.LogRecord(
-        "This is a message with 17775554324, and also 18884449323 and also 17775554324"
+        name="log thing",
-    )
+        level="info",
-    assert (
+        pathname="path",
-        result
+        lineno=123,
-        == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324"
+        msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov",
+        exc_info=None,
+        args=None,
     )
+    pii_filter = logging.PIIFilter()
+    clean_msg = "phone1: 1XXXXXXXXXX, phone2: 1XXXXXXXXXX, email1: XXXXX@XXXXXXX, email2: XXXXX@XXXXXXX"
+    assert pii_filter.filter(record).msg == clean_msg