darkhelm/notifications-admin
1
0
Fork 0
mirror of https://github.com/GSA/notifications-admin.git synced 2026-06-19 04:36:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/GSA/notifications-admin into 1484-dashboard-visualizations

Browse Source
This commit is contained in:
Jonathan Bobel
2024-06-27 11:03:27 -04:00
parent 149c23b721 8350620c76
commit 0a17aaad0d
29 changed files with 324 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.ds.baseline
View File

@@ -169,7 +169,7 @@
"filename": "app/config.py",
"hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
"is_verified": false,
"line_number": 111,
"line_number": 117,
"is_secret": false
}
],

8
.github/workflows/deploy-demo.yml vendored
View File

@@ -18,11 +18,11 @@ jobs:
- name: Check for changes to Terraform
id: changed-terraform-files
uses: tj-actions/changed-files@v41.0.0
uses: tj-actions/changed-files@v44
with:
files: |
terraform/demo
terraform/shared
terraform/demo/**
terraform/shared/**
.github/workflows/deploy-demo.yml
- name: Terraform init
if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
- name: Check for changes to egress config
id: changed-egress-config
uses: tj-actions/changed-files@v41.0.0
uses: tj-actions/changed-files@v44
with:
files: |
deploy-config/egress_proxy/notify-admin-demo.*.acl

8
.github/workflows/deploy-prod.yml vendored
View File

@@ -18,11 +18,11 @@ jobs:
- name: Check for changes to Terraform
id: changed-terraform-files
uses: tj-actions/changed-files@v41.0.0
uses: tj-actions/changed-files@v44
with:
files: |
terraform/production
terraform/shared
terraform/production/**
terraform/shared/**
.github/workflows/deploy-prod.yml
- name: Terraform init
if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
- name: Check for changes to egress config
id: changed-egress-config
uses: tj-actions/changed-files@v41.0.0
uses: tj-actions/changed-files@v44
with:
files: |
deploy-config/egress_proxy/notify-admin-production.*.acl

166
.github/workflows/deploy.yml vendored
View File

@@ -17,96 +17,96 @@ jobs:
environment: staging
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Check for changes to Terraform
id: changed-terraform-files
uses: tj-actions/changed-files@v41.0.0
with:
files: |
terraform/staging
terraform/shared
.github/workflows/deploy.yml
- name: Terraform init
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
run: terraform init
- name: Terraform apply
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
run: terraform apply -auto-approve -input=false
- name: Check for changes to Terraform
id: changed-terraform-files
uses: tj-actions/changed-files@v44
with:
files: |
terraform/staging/**
terraform/shared/**
.github/workflows/deploy.yml
- name: Terraform init
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
run: terraform init
- name: Terraform apply
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
run: terraform apply -auto-approve -input=false
- uses: ./.github/actions/setup-project
- uses: ./.github/actions/setup-project
- name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt
- name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt
- name: Deploy to cloud.gov
uses: 18f/cg-deploy-action@main
env:
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
COMMIT_HASH: ${{ github.sha }}
LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
with:
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
cf_org: gsa-tts-benefits-studio
cf_space: notify-staging
push_arguments: >-
--vars-file deploy-config/staging.yml
--var DANGEROUS_SALT="$DANGEROUS_SALT"
--var SECRET_KEY="$SECRET_KEY"
--var ADMIN_CLIENT_USERNAME="notify-admin"
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
--var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
--var NR_BROWSER_KEY="$NR_BROWSER_KEY"
--var COMMIT_HASH="$COMMIT_HASH"
--var LOGIN_PEM="$LOGIN_PEM"
--var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
--var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
--var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
--var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
--var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
--var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
--var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
- name: Deploy to cloud.gov
uses: 18f/cg-deploy-action@main
env:
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
COMMIT_HASH: ${{ github.sha }}
LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
with:
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
cf_org: gsa-tts-benefits-studio
cf_space: notify-staging
push_arguments: >-
--vars-file deploy-config/staging.yml
--var DANGEROUS_SALT="$DANGEROUS_SALT"
--var SECRET_KEY="$SECRET_KEY"
--var ADMIN_CLIENT_USERNAME="notify-admin"
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
--var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
--var NR_BROWSER_KEY="$NR_BROWSER_KEY"
--var COMMIT_HASH="$COMMIT_HASH"
--var LOGIN_PEM="$LOGIN_PEM"
--var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
--var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
--var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
--var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
--var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
--var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
--var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
- name: Check for changes to egress config
id: changed-egress-config
uses: tj-actions/changed-files@v41.0.0
with:
files: |
deploy-config/egress_proxy/notify-admin-staging.*.acl
.github/actions/deploy-proxy/action.yml
.github/workflows/deploy.yml
- name: Deploy egress proxy
if: steps.changed-egress-config.outputs.any_changed == 'true'
uses: ./.github/actions/deploy-proxy
with:
cf_space: notify-staging
app: notify-admin-staging
- name: Check for changes to egress config
id: changed-egress-config
uses: tj-actions/changed-files@v44
with:
files: |
deploy-config/egress_proxy/notify-admin-staging.*.acl
.github/actions/deploy-proxy/action.yml
.github/workflows/deploy.yml
- name: Deploy egress proxy
if: steps.changed-egress-config.outputs.any_changed == 'true'
uses: ./.github/actions/deploy-proxy
with:
cf_space: notify-staging
app: notify-admin-staging
bail:
runs-on: ubuntu-latest

28
README.md
View File

@@ -40,7 +40,7 @@ You will need the following items:
[Follow the instructions here to set up the Notify.gov API.](https://github.com/GSA/notifications-api#before-you-start)
The Notify.gov API is required in order for the Notify.gov Admin UI to run, and
it will also take care of many of the steps that are listed here. The sections
it will also take care of many of the steps that are listed here. The sections
that are a repeat from the API setup are flagged with an **[API Step]** label
in front of them.
@@ -83,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these
locations:
For BASH shells:
- `~/.bashrc`
- `~/.bash_profile`
- `~/.profile`
For ZSH shells:
- `~/.zshrc`
- `~/.zprofile`
@@ -97,7 +99,7 @@ environments.
Which file you need to modify depends on whether or not you are running an
interactive shell or a login shell
(see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell)
for an explanation of the differences). If you're still not sure, please ask
for an explanation of the differences). If you're still not sure, please ask
the team for help!
Once you determine which file you'll need to modify, add these lines before any
@@ -158,7 +160,7 @@ _NOTE: This project currently uses the latest `1.4.x release of Terraform._
#### [API Step] Python Installation
Now we're going to install a tool to help us manage Python versions and
virtual environments on our system. First, we'll install
virtual environments on our system. First, we'll install
[pyenv](https://github.com/pyenv/pyenv) and one of its plugins,
[pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew:
@@ -285,7 +287,7 @@ we'll use `3.12` in our example here since we recently upgraded to this version:
pyenv install 3.12
```
Next, delete the virtual environment you previously had set up. If you followed
Next, delete the virtual environment you previously had set up. If you followed
the instructions above with the first-time set up, you can do this with `pyenv`:
```sh
@@ -306,6 +308,20 @@ you'll be set with an upgraded version of Python.
_If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
#### Updating the .env file for Login.gov
To configure the application for Login.gov, you will need to update the following environment variables in the .env file:
```
COMMIT_HASH=”--------”
```
Reach out to someone on the team to get the most recent Login.gov key.
```
LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE"
```
#### Updating the .env file for E2E tests
With the newly created `.env` file in place, you'll need to make one more
@@ -353,7 +369,7 @@ API is running as well!
## Creating a 'First User' in the database
After you have completed all setup steps, you will be unable to log in, because there
will not be a user in the database to link to the login.gov account you are using. So
will not be a user in the database to link to the login.gov account you are using. So
you will need to create that user in your database using the 'create-test-user' command.
Open two terminals pointing to the api project and then run these commands in the
@@ -372,8 +388,6 @@ is the same one you are using in login.gov and make sure your phone number is in
If for any reason in the course of development it is necessary for your to delete your db
via the `dropdb` command, you will need to repeat these steps when you recreate your db.
## Git Hooks
We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to

8
app/config.py
View File

@@ -53,7 +53,13 @@ class Config(object):
PERMANENT_SESSION_LIFETIME = 1800 # 30 Minutes
SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year
REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45
ACTIVITY_STATS_LIMIT_DAYS = 7
ACTIVITY_STATS_LIMIT_DAYS = {
"today": 0,
"one_day": 1,
"three_day": 3,
"five_day": 5,
"seven_day": 7,
}
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_NAME = "notify_admin_session"
SESSION_COOKIE_SECURE = True

39
app/main/views/jobs.py
View File

@@ -143,11 +143,40 @@ def view_notifications(service_id, message_type=None):
True: ["reference"],
False: [],
}.get(bool(current_service.api_keys)),
download_link=url_for(
download_link_one_day=url_for(
".download_notifications_csv",
service_id=current_service.id,
message_type=message_type,
status=request.args.get("status"),
number_of_days="one_day",
),
download_link_today=url_for(
".download_notifications_csv",
service_id=current_service.id,
message_type=message_type,
status=request.args.get("status"),
number_of_days="today",
),
download_link_three_day=url_for(
".download_notifications_csv",
service_id=current_service.id,
message_type=message_type,
status=request.args.get("status"),
number_of_days="three_day",
),
download_link_five_day=url_for(
".download_notifications_csv",
service_id=current_service.id,
message_type=message_type,
status=request.args.get("status"),
number_of_days="five_day",
),
download_link_seven_day=url_for(
".download_notifications_csv",
service_id=current_service.id,
message_type=message_type,
status=request.args.get("status"),
number_of_days="seven_day",
),
)
@@ -183,10 +212,9 @@ def get_notifications(service_id, message_type, status_override=None): # noqa
filter_args["status"] = set_status_filters(filter_args)
service_data_retention_days = None
search_term = request.form.get("to", "")
if message_type is not None:
service_data_retention_days = current_service.get_days_of_retention(
message_type
message_type, number_of_days="seven_day"
)
if request.path.endswith("csv") and current_user.has_permissions("view_activity"):
@@ -212,7 +240,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa
)
url_args = {"message_type": message_type, "status": request.args.get("status")}
prev_page = None
if "links" in notifications and notifications["links"].get("prev", None):
prev_page = generate_previous_dict(
"main.view_notifications", service_id, page, url_args=url_args
@@ -233,7 +260,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa
)
else:
download_link = None
return {
"service_data_retention_days": service_data_retention_days,
"counts": render_template(
@@ -362,6 +388,7 @@ def get_job_partials(job):
filter_args = parse_filter_args(request.args)
filter_args["status"] = set_status_filters(filter_args)
notifications = job.get_notifications(status=filter_args["status"])
number_of_days = "seven_day"
counts = render_template(
"partials/count.html",
counts=_get_job_counts(job),
@@ -371,7 +398,7 @@ def get_job_partials(job):
),
)
service_data_retention_days = current_service.get_days_of_retention(
job.template_type
job.template_type, number_of_days
)
if request.referrer is not None:

4
app/main/views/notifications.py
View File

@@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification):
def download_notifications_csv(service_id):
filter_args = parse_filter_args(request.args)
filter_args["status"] = set_status_filters(filter_args)
number_of_days = request.args["number_of_days"]
service_data_retention_days = current_service.get_days_of_retention(
filter_args.get("message_type")[0]
filter_args.get("message_type")[0], number_of_days
)
file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
file_time = f"{file_time} {get_user_preferred_timezone()}"

30
app/main/views/send.py
View File

@@ -51,7 +51,6 @@ from app.utils.templates import get_template
from app.utils.user import user_has_permissions
from notifications_utils import SMS_CHAR_COUNT_LIMIT
from notifications_utils.insensitive_dict import InsensitiveDict
from notifications_utils.logging import scrub
from notifications_utils.recipients import RecipientCSV, first_column_headings
from notifications_utils.sanitise_text import SanitiseASCII
@@ -953,9 +952,6 @@ def send_notification(service_id, template_id):
)
)
current_app.logger.info(
hilite(scrub(f"Recipient for the one-off will be {recipient}"))
)
keys = []
values = []
for k, v in session["placeholders"].items():
@@ -971,6 +967,12 @@ def send_notification(service_id, template_id):
)
my_data = {"filename": filename, "template_id": template_id, "data": data}
upload_id = s3upload(service_id, my_data)
# To debug messages that the user reports have not been sent, we log
# the csv filename and the job id. The user will give us the file name,
# so we can search on that to obtain the job id, which we can use elsewhere
# on the API side to find out what happens to the message.
current_app.logger.info(hilite(f"One-off file: {filename} job_id: {upload_id}"))
form = CsvUploadForm()
form.file.data = my_data
form.file.name = filename
@@ -989,19 +991,6 @@ def send_notification(service_id, template_id):
valid="True",
)
# Here we are attempting to cleverly link the job id to the one-off recipient
# If we know the partial phone number of the recipient, we can search
# on that initially and find this, which will give us the job_id
# And once we know the job_id, we can search on that and it might tell us something
# about report generation.
current_app.logger.info(
hilite(
scrub(
f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}"
)
)
)
session.pop("recipient")
session.pop("placeholders")
@@ -1033,7 +1022,12 @@ def send_notification(service_id, template_id):
job_id=upload_id,
)
)
total = notifications["total"]
current_app.logger.info(
hilite(
f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}"
)
)
return redirect(
url_for(
".view_job",

7
app/models/service.py
View File

@@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin):
def get_data_retention_item(self, id):
return next((dr for dr in self.data_retention if dr["id"] == id), None)
def get_days_of_retention(self, notification_type):
def get_days_of_retention(self, notification_type, number_of_days):
return next(
(
dr
@@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin):
if dr["notification_type"] == notification_type
),
{},
).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"])
).get(
"days_of_retention",
current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days),
)
@cached_property
def organization(self):

14
app/templates/views/notifications.html
View File

@@ -64,10 +64,22 @@
{% if current_user.has_permissions('view_activity') %}
<p class="font-body-sm">
<a href="{{ download_link }}" download="download" class="usa-link">Download this report (<abbr title="Comma separated values">CSV</abbr>)</a>
<a href="{{ download_link_seven_day }}" download="download" class="usa-link">Download all data last 7 days (<abbr title="Comma separated values">CSV</abbr>)</a>
&emsp;
Data available for {{ partials.service_data_retention_days }} days
</p>
<p class="font-body-sm">
<a href="{{ download_link_five_day }}" download="download" class="usa-link">Download all data last 5 days (<abbr title="Comma separated values">CSV</abbr>)</a>
&emsp;
</p>
<p class="font-body-sm">
<a href="{{ download_link_three_day }}" download="download" class="usa-link">Download all data last 3 days (<abbr title="Comma separated values">CSV</abbr>)</a>
&emsp;
</p>
<p class="font-body-sm">
<a href="{{ download_link_today }}" download="download" class="usa-link">Download all data today (<abbr title="Comma separated values">CSV</abbr>)</a>
&emsp;
</p>
{% endif %}
{{ ajax_block(

6
app/utils/csv.py
View File

@@ -7,7 +7,6 @@ from flask_login import current_user
from app.models.spreadsheet import Spreadsheet
from app.utils import hilite
from app.utils.templates import get_sample_template
from notifications_utils.logging import scrub
from notifications_utils.recipients import RecipientCSV
@@ -74,12 +73,11 @@ def generate_notifications_csv(**kwargs):
# This generates the "batch" csv report
if kwargs.get("job_id"):
# The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug
# Some unit tests are mocking the kwargs and turning them into a function instead of dict,
# hence the try/except.
try:
current_app.logger.info(
hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}")
hilite(f"Setting up report with kwargs {json.dumps(kwargs)}")
)
except TypeError:
pass
@@ -89,7 +87,7 @@ def generate_notifications_csv(**kwargs):
# we display to 999 characters, because we don't want to show the contents for reports with thousands of rows.
current_app.logger.info(
hilite(
f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}"
f"Original csv for job_id {kwargs['job_id']}: {original_file_contents[0:999]}"
)
)
original_upload = RecipientCSV(

2
deploy-config/production.yml
View File

@@ -1,6 +1,6 @@
env: production
instances: 2
memory: 1G
memory: 2G
public_admin_route: beta.notify.gov
cloud_dot_gov_route: notify.app.cloud.gov
redis_enabled: 1

25
gunicorn_config.py
View File

@@ -5,7 +5,10 @@ import multiprocessing
import gunicorn
# Let gunicorn figure out the right number of workers
workers = multiprocessing.cpu_count() * 2 + 1
# The recommended formula is cpu_count() * 2 + 1
# but we have an unusual configuration with a lot of cpus and not much memory
# so adjust it.
workers = multiprocessing.cpu_count()
worker_class = "eventlet"
bind = "0.0.0.0:{}".format(os.getenv("PORT"))
disable_redirect_access_to_syslog = True
@@ -16,23 +19,3 @@ def worker_abort(worker):
worker.log.info("worker received ABORT")
for stack in sys._current_frames().values():
worker.log.error("".join(traceback.format_stack(stack)))
# This issue is fixed in the 22.0.0 release, which we are using
# See github issue for details
# def fix_ssl_monkeypatching():
# """
# eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently
# a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However
# eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not
# ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type
# :scream:
# https://github.com/eventlet/eventlet/issues/692
# """
# # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again.
# # eventlet.monkey_patch can be called multiple times without issue
# eventlet.monkey_patch()
# eventlet.green.ssl.timeout_exc = socket.timeout
# fix_ssl_monkeypatching()

2
manifest.yml
View File

@@ -12,7 +12,7 @@ applications:
- route: ((cloud_dot_gov_route))
services:
- notify-admin-redis-((env))
- notify-admin-redis-v70-((env))
- notify-api-csv-upload-bucket-((env))
- notify-admin-logo-upload-bucket-((env))

38
notifications_utils/logging.py
View File

@@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter):
handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"]))
handler.addFilter(RequestIdFilter())
handler.addFilter(ServiceIdFilter())
handler.addFilter(PIIFilter())
return handler
@@ -134,13 +135,30 @@ class JSONFormatter(BaseJSONFormatter):
return log_record
def scrub(msg):
# Eventually we want to scrub all messages in all logs for phone numbers
# and email addresses, masking them. Ultimately this will probably get
# refactored into a 'SafeLogger' subclass or something, but let's start here
# with phones.
phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
for phone in phones:
msg = msg.replace(phone, f"1XXXXX{phone[-5:]}")
return msg
class PIIFilter(logging.Filter):
def scrub(self, msg):
# Eventually we want to scrub all messages in all logs for phone numbers
# and email addresses, masking them. Ultimately this will probably get
# refactored into a 'SafeLogger' subclass or something, but let's start here
# with phones.
# Sometimes just an exception object is passed in for the message, skip those.
if not isinstance(msg, str):
return msg
phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
for phone in phones:
msg = msg.replace(phone, "1XXXXXXXXXX")
emails = re.findall(
r"[\w\.-]+@[\w\.-]+", msg
) # ['alice@google.com', 'bob@abc.com']
for email in emails:
# do something with each found email string
masked_email = "XXXXX@XXXXXXX"
msg = msg.replace(email, masked_email)
return msg
def filter(self, record):
record.msg = self.scrub(record.msg)
return record

4
terraform/bootstrap/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}
}

16
terraform/demo/main.tf
View File

@@ -6,7 +6,7 @@ locals {
recursive_delete = false
}
module "redis" {
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
cf_org_name = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
redis_plan_name = "redis-dev"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-dev"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "logo_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"

4
terraform/demo/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}

4
terraform/development/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}
}

16
terraform/production/main.tf
View File

@@ -6,7 +6,7 @@ locals {
recursive_delete = false
}
module "redis" {
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
cf_org_name = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
redis_plan_name = "redis-3node-large"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-3node-large"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "logo_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"

4
terraform/production/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}

16
terraform/sandbox/main.tf
View File

@@ -6,7 +6,7 @@ locals {
recursive_delete = true
}
module "redis" {
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
cf_org_name = local.cf_org_name
@@ -16,6 +16,20 @@ module "redis" {
redis_plan_name = "redis-dev"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-dev"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "logo_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"

4
terraform/sandbox/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}

4
terraform/shared/container_networking/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}
}

24
terraform/staging/main.tf
View File

@@ -6,7 +6,15 @@ locals {
recursive_delete = true
}
module "redis" {
resource "null_resource" "prevent_destroy" {
lifecycle {
prevent_destroy = false # destroying staging is allowed
}
}
module "redis" { # default v6.2; delete after v7.0 resource is bound
source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
cf_org_name = local.cf_org_name
@@ -16,6 +24,20 @@ module "redis" {
redis_plan_name = "redis-dev"
}
module "redis-v70" {
source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0"
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
name = "${local.app_name}-redis-v70-${local.env}"
redis_plan_name = "redis-dev"
json_params = jsonencode(
{
"engineVersion" : "7.0",
}
)
}
module "logo_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"

4
terraform/staging/providers.tf
View File

@@ -1,9 +1,9 @@
terraform {
required_version = "~> 1.0"
required_version = "~> 1.7"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.53.0"
version = "0.53.1"
}
}

14
tests/app/main/views/test_activity.py
View File

@@ -228,12 +228,18 @@ def test_can_show_notifications_if_data_retention_not_available(
url_for,
".download_notifications_csv",
message_type=None,
number_of_days="seven_day",
),
),
(
create_active_user_with_permissions(),
{"status": "failed"},
partial(url_for, ".download_notifications_csv", status="failed"),
partial(
url_for,
".download_notifications_csv",
status="failed",
number_of_days="seven_day",
),
),
(
create_active_user_with_permissions(),
@@ -242,15 +248,13 @@ def test_can_show_notifications_if_data_retention_not_available(
url_for,
".download_notifications_csv",
message_type="sms",
number_of_days="seven_day",
),
),
(
create_active_user_view_permissions(),
{},
partial(
url_for,
".download_notifications_csv",
),
partial(url_for, ".download_notifications_csv", number_of_days="seven_day"),
),
(
create_active_caseworking_user(),

19
tests/notifications_utils/test_logging.py
View File

@@ -51,11 +51,16 @@ def test_base_json_formatter_contains_service_id():
assert service_id_filter.filter(record).service_id == "no-service-id"
def test_scrub():
result = logging.scrub(
"This is a message with 17775554324, and also 18884449323 and also 17775554324"
)
assert (
result
== "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324"
def test_pii_filter():
record = builtin_logging.LogRecord(
name="log thing",
level="info",
pathname="path",
lineno=123,
msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov",
exc_info=None,
args=None,
)
pii_filter = logging.PIIFilter()
clean_msg = "phone1: 1XXXXXXXXXX, phone2: 1XXXXXXXXXX, email1: XXXXX@XXXXXXX, email2: XXXXX@XXXXXXX"
assert pii_filter.filter(record).msg == clean_msg