Merge branch 'main' of https://github.com/GSA/notifications-admin into 1484-dashboard-visualizations

.ds.baseline

@@ -169,7 +169,7 @@
         "filename": "app/config.py",
         "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
         "is_verified": false,
-        "line_number": 111,
+        "line_number": 117,
         "is_secret": false
       }
     ],

.github/workflows/deploy-demo.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/demo
+            terraform/demo/**
-            terraform/shared
+            terraform/shared/**
             .github/workflows/deploy-demo.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-demo.*.acl

.github/workflows/deploy-prod.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/production
+            terraform/production/**
-            terraform/shared
+            terraform/shared/**
             .github/workflows/deploy-prod.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-production.*.acl

.github/workflows/deploy.yml

@@ -23,11 +23,11 @@ jobs:
 
     - name: Check for changes to Terraform
       id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+      uses: tj-actions/changed-files@v44
       with:
         files: |
-            terraform/staging
+          terraform/staging/**
-            terraform/shared
+          terraform/shared/**
           .github/workflows/deploy.yml
     - name: Terraform init
       if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -95,7 +95,7 @@ jobs:
 
     - name: Check for changes to egress config
       id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+      uses: tj-actions/changed-files@v44
       with:
         files: |
           deploy-config/egress_proxy/notify-admin-staging.*.acl

README.md

@@ -83,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these
 locations:
 
 For BASH shells:
+
 - `~/.bashrc`
 - `~/.bash_profile`
 - `~/.profile`
 
 For ZSH shells:
+
 - `~/.zshrc`
 - `~/.zprofile`
 
@@ -306,6 +308,20 @@ you'll be set with an upgraded version of Python.
 
 _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
 
+#### Updating the .env file for Login.gov
+
+To configure the application for Login.gov, you will need to update the following environment variables in the .env file:
+
+```
+COMMIT_HASH=”--------”
+```
+
+Reach out to someone on the team to get the most recent Login.gov key.
+
+```
+LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE"
+```
+
 #### Updating the .env file for E2E tests
 
 With the newly created `.env` file in place, you'll need to make one more
@@ -372,8 +388,6 @@ is the same one you are using in login.gov and make sure your phone number is in
 If for any reason in the course of development it is necessary for your to delete your db
 via the `dropdb` command, you will need to repeat these steps when you recreate your db.
 
-
-
 ## Git Hooks
 
 We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to

app/config.py

@@ -53,7 +53,13 @@ class Config(object):
     PERMANENT_SESSION_LIFETIME = 1800  # 30 Minutes
     SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60  # 1 year
     REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45
-    ACTIVITY_STATS_LIMIT_DAYS = 7
+    ACTIVITY_STATS_LIMIT_DAYS = {
+        "today": 0,
+        "one_day": 1,
+        "three_day": 3,
+        "five_day": 5,
+        "seven_day": 7,
+    }
     SESSION_COOKIE_HTTPONLY = True
     SESSION_COOKIE_NAME = "notify_admin_session"
     SESSION_COOKIE_SECURE = True

app/main/views/jobs.py

@@ -143,11 +143,40 @@ def view_notifications(service_id, message_type=None):
             True: ["reference"],
             False: [],
         }.get(bool(current_service.api_keys)),
-        download_link=url_for(
+        download_link_one_day=url_for(
             ".download_notifications_csv",
             service_id=current_service.id,
             message_type=message_type,
             status=request.args.get("status"),
+            number_of_days="one_day",
+        ),
+        download_link_today=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="today",
+        ),
+        download_link_three_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="three_day",
+        ),
+        download_link_five_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="five_day",
+        ),
+        download_link_seven_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="seven_day",
         ),
     )
 
@@ -183,10 +212,9 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     filter_args["status"] = set_status_filters(filter_args)
     service_data_retention_days = None
     search_term = request.form.get("to", "")
-
     if message_type is not None:
         service_data_retention_days = current_service.get_days_of_retention(
-            message_type
+            message_type, number_of_days="seven_day"
         )
 
     if request.path.endswith("csv") and current_user.has_permissions("view_activity"):
@@ -212,7 +240,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     )
     url_args = {"message_type": message_type, "status": request.args.get("status")}
     prev_page = None
-
     if "links" in notifications and notifications["links"].get("prev", None):
         prev_page = generate_previous_dict(
             "main.view_notifications", service_id, page, url_args=url_args
@@ -233,7 +260,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
         )
     else:
         download_link = None
-
     return {
         "service_data_retention_days": service_data_retention_days,
         "counts": render_template(
@@ -362,6 +388,7 @@ def get_job_partials(job):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
     notifications = job.get_notifications(status=filter_args["status"])
+    number_of_days = "seven_day"
     counts = render_template(
         "partials/count.html",
         counts=_get_job_counts(job),
@@ -371,7 +398,7 @@ def get_job_partials(job):
         ),
     )
     service_data_retention_days = current_service.get_days_of_retention(
-        job.template_type
+        job.template_type, number_of_days
     )
 
     if request.referrer is not None:

app/main/views/notifications.py

@@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification):
 def download_notifications_csv(service_id):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
-
+    number_of_days = request.args["number_of_days"]
     service_data_retention_days = current_service.get_days_of_retention(
-        filter_args.get("message_type")[0]
+        filter_args.get("message_type")[0], number_of_days
     )
     file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
     file_time = f"{file_time} {get_user_preferred_timezone()}"

app/main/views/send.py

@@ -51,7 +51,6 @@ from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
 from notifications_utils.insensitive_dict import InsensitiveDict
-from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV, first_column_headings
 from notifications_utils.sanitise_text import SanitiseASCII
 
@@ -953,9 +952,6 @@ def send_notification(service_id, template_id):
             )
         )
 
-    current_app.logger.info(
-        hilite(scrub(f"Recipient for the one-off will be {recipient}"))
-    )
     keys = []
     values = []
     for k, v in session["placeholders"].items():
@@ -971,6 +967,12 @@ def send_notification(service_id, template_id):
     )
     my_data = {"filename": filename, "template_id": template_id, "data": data}
     upload_id = s3upload(service_id, my_data)
+
+    # To debug messages that the user reports have not been sent, we log
+    # the csv filename and the job id.  The user will give us the file name,
+    # so we can search on that to obtain the job id, which we can use elsewhere
+    # on the API side to find out what happens to the message.
+    current_app.logger.info(hilite(f"One-off file: {filename} job_id: {upload_id}"))
     form = CsvUploadForm()
     form.file.data = my_data
     form.file.name = filename
@@ -989,19 +991,6 @@ def send_notification(service_id, template_id):
         valid="True",
     )
 
-    # Here we are attempting to cleverly link the job id to the one-off recipient
-    # If we know the partial phone number of the recipient, we can search
-    # on that initially and find this, which will give us the job_id
-    # And once we know the job_id, we can search on that and it might tell us something
-    # about report generation.
-    current_app.logger.info(
-        hilite(
-            scrub(
-                f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}"
-            )
-        )
-    )
-
     session.pop("recipient")
     session.pop("placeholders")
 
@@ -1033,7 +1022,12 @@ def send_notification(service_id, template_id):
                 job_id=upload_id,
             )
         )
-
+    total = notifications["total"]
+    current_app.logger.info(
+        hilite(
+            f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}"
+        )
+    )
     return redirect(
         url_for(
             ".view_job",

app/models/service.py

@@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin):
     def get_data_retention_item(self, id):
         return next((dr for dr in self.data_retention if dr["id"] == id), None)
 
-    def get_days_of_retention(self, notification_type):
+    def get_days_of_retention(self, notification_type, number_of_days):
         return next(
             (
                 dr
@@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin):
                 if dr["notification_type"] == notification_type
             ),
             {},
-        ).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"])
+        ).get(
+            "days_of_retention",
+            current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days),
+        )
 
     @cached_property
     def organization(self):

app/templates/views/notifications.html

@@ -64,10 +64,22 @@
 
   {% if current_user.has_permissions('view_activity') %}
     <p class="font-body-sm">
-      <a href="{{ download_link }}" download="download" class="usa-link">Download this report (<abbr title="Comma separated values">CSV</abbr>)</a>
+      <a href="{{ download_link_seven_day }}" download="download" class="usa-link">Download all data last 7 days (<abbr title="Comma separated values">CSV</abbr>)</a>
       &emsp;
       Data available for {{ partials.service_data_retention_days }} days
     </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_five_day }}" download="download" class="usa-link">Download all data last 5 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_three_day }}" download="download" class="usa-link">Download all data last 3 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_today }}" download="download" class="usa-link">Download all data today (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
   {% endif %}
 
   {{ ajax_block(

app/utils/csv.py

@@ -7,7 +7,6 @@ from flask_login import current_user
 from app.models.spreadsheet import Spreadsheet
 from app.utils import hilite
 from app.utils.templates import get_sample_template
-from notifications_utils.logging import scrub
 from notifications_utils.recipients import RecipientCSV
 
 
@@ -74,12 +73,11 @@ def generate_notifications_csv(**kwargs):
 
     # This generates the "batch" csv report
     if kwargs.get("job_id"):
-        # The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug
         # Some unit tests are mocking the kwargs and turning them into a function instead of dict,
         # hence the try/except.
         try:
             current_app.logger.info(
-                hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}")
+                hilite(f"Setting up report with kwargs {json.dumps(kwargs)}")
             )
         except TypeError:
             pass
@@ -89,7 +87,7 @@ def generate_notifications_csv(**kwargs):
         # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows.
         current_app.logger.info(
             hilite(
-                f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}"
+                f"Original csv for job_id {kwargs['job_id']}: {original_file_contents[0:999]}"
             )
         )
         original_upload = RecipientCSV(

deploy-config/production.yml

@@ -1,6 +1,6 @@
 env: production
 instances: 2
-memory: 1G
+memory: 2G
 public_admin_route: beta.notify.gov
 cloud_dot_gov_route: notify.app.cloud.gov
 redis_enabled: 1

gunicorn_config.py

@@ -5,7 +5,10 @@ import multiprocessing
 import gunicorn
 
 # Let gunicorn figure out the right number of workers
-workers = multiprocessing.cpu_count() * 2 + 1
+# The recommended formula is cpu_count() * 2 + 1
+# but we have an unusual configuration with a lot of cpus and not much memory
+# so adjust it.
+workers = multiprocessing.cpu_count()
 worker_class = "eventlet"
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))
 disable_redirect_access_to_syslog = True
@@ -16,23 +19,3 @@ def worker_abort(worker):
     worker.log.info("worker received ABORT")
     for stack in sys._current_frames().values():
         worker.log.error("".join(traceback.format_stack(stack)))
-
-
-# This issue is fixed in the 22.0.0 release, which we are using
-# See github issue for details
-# def fix_ssl_monkeypatching():
-#     """
-#     eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently
-#     a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However
-#     eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not
-#     ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type
-#     :scream:
-#     https://github.com/eventlet/eventlet/issues/692
-#     """
-#     # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again.
-#     # eventlet.monkey_patch can be called multiple times without issue
-#     eventlet.monkey_patch()
-#     eventlet.green.ssl.timeout_exc = socket.timeout
-
-
-# fix_ssl_monkeypatching()

manifest.yml

@@ -12,7 +12,7 @@ applications:
       - route: ((cloud_dot_gov_route))
 
     services:
-      - notify-admin-redis-((env))
+      - notify-admin-redis-v70-((env))
       - notify-api-csv-upload-bucket-((env))
       - notify-admin-logo-upload-bucket-((env))
 

notifications_utils/logging.py

@@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter):
     handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"]))
     handler.addFilter(RequestIdFilter())
     handler.addFilter(ServiceIdFilter())
+    handler.addFilter(PIIFilter())
 
     return handler
 
@@ -134,13 +135,30 @@ class JSONFormatter(BaseJSONFormatter):
         return log_record
 
 
-def scrub(msg):
+class PIIFilter(logging.Filter):
+    def scrub(self, msg):
         # Eventually we want to scrub all messages in all logs for phone numbers
         # and email addresses, masking them.  Ultimately this will probably get
         # refactored into a 'SafeLogger' subclass or something, but let's start here
         # with phones.
+
+        # Sometimes just an exception object is passed in for the message, skip those.
+        if not isinstance(msg, str):
+            return msg
         phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
         phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
         for phone in phones:
-        msg = msg.replace(phone, f"1XXXXX{phone[-5:]}")
+            msg = msg.replace(phone, "1XXXXXXXXXX")
+
+        emails = re.findall(
+            r"[\w\.-]+@[\w\.-]+", msg
+        )  # ['alice@google.com', 'bob@abc.com']
+        for email in emails:
+            # do something with each found email string
+            masked_email = "XXXXX@XXXXXXX"
+            msg = msg.replace(email, masked_email)
         return msg
+
+    def filter(self, record):
+        record.msg = self.scrub(record.msg)
+        return record

terraform/bootstrap/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/demo/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = false
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/demo/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/development/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/production/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = false
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-3node-large"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-3node-large"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/production/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/sandbox/main.tf

@@ -6,7 +6,7 @@ locals {
   recursive_delete = true
 }
 
-module "redis" {
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/sandbox/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/shared/container_networking/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/staging/main.tf

@@ -6,7 +6,15 @@ locals {
   recursive_delete = true
 }
 
-module "redis" {
+resource "null_resource" "prevent_destroy" {
+
+  lifecycle {
+    prevent_destroy = false # destroying staging is allowed
+  }
+}
+
+
+module "redis" { # default v6.2; delete after v7.0 resource is bound
   source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
 
   cf_org_name      = local.cf_org_name
@@ -16,6 +24,20 @@ module "redis" {
   redis_plan_name  = "redis-dev"
 }
 
+module "redis-v70" {
+  source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
+
+  cf_org_name     = local.cf_org_name
+  cf_space_name   = local.cf_space_name
+  name            = "${local.app_name}-redis-v70-${local.env}"
+  redis_plan_name = "redis-dev"
+  json_params = jsonencode(
+    {
+      "engineVersion" : "7.0",
+    }
+  )
+}
+
 module "logo_upload_bucket" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
 

terraform/staging/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

tests/app/main/views/test_activity.py

@@ -228,12 +228,18 @@ def test_can_show_notifications_if_data_retention_not_available(
                 url_for,
                 ".download_notifications_csv",
                 message_type=None,
+                number_of_days="seven_day",
             ),
         ),
         (
             create_active_user_with_permissions(),
             {"status": "failed"},
-            partial(url_for, ".download_notifications_csv", status="failed"),
+            partial(
+                url_for,
+                ".download_notifications_csv",
+                status="failed",
+                number_of_days="seven_day",
+            ),
         ),
         (
             create_active_user_with_permissions(),
@@ -242,15 +248,13 @@ def test_can_show_notifications_if_data_retention_not_available(
                 url_for,
                 ".download_notifications_csv",
                 message_type="sms",
+                number_of_days="seven_day",
             ),
         ),
         (
             create_active_user_view_permissions(),
             {},
-            partial(
+            partial(url_for, ".download_notifications_csv", number_of_days="seven_day"),
-                url_for,
-                ".download_notifications_csv",
-            ),
         ),
         (
             create_active_caseworking_user(),

tests/notifications_utils/test_logging.py

@@ -51,11 +51,16 @@ def test_base_json_formatter_contains_service_id():
     assert service_id_filter.filter(record).service_id == "no-service-id"
 
 
-def test_scrub():
+def test_pii_filter():
-    result = logging.scrub(
+    record = builtin_logging.LogRecord(
-        "This is a message with 17775554324, and also 18884449323 and also 17775554324"
+        name="log thing",
-    )
+        level="info",
-    assert (
+        pathname="path",
-        result
+        lineno=123,
-        == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324"
+        msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov",
+        exc_info=None,
+        args=None,
     )
+    pii_filter = logging.PIIFilter()
+    clean_msg = "phone1: 1XXXXXXXXXX, phone2: 1XXXXXXXXXX, email1: XXXXX@XXXXXXX, email2: XXXXX@XXXXXXX"
+    assert pii_filter.filter(record).msg == clean_msg