diff --git a/.ds.baseline b/.ds.baseline index 82ab59b8d..bb6b3ce39 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -169,7 +169,7 @@ "filename": "app/config.py", "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc", "is_verified": false, - "line_number": 111, + "line_number": 117, "is_secret": false } ], diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index dc725f157..89adc1f29 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -18,11 +18,11 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | - terraform/demo - terraform/shared + terraform/demo/** + terraform/shared/** .github/workflows/deploy-demo.yml - name: Terraform init if: steps.changed-terraform-files.outputs.any_changed == 'true' @@ -88,7 +88,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-admin-demo.*.acl diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index d614bf309..262079be8 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -18,11 +18,11 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | - terraform/production - terraform/shared + terraform/production/** + terraform/shared/** .github/workflows/deploy-prod.yml - name: Terraform init if: steps.changed-terraform-files.outputs.any_changed == 'true' @@ -88,7 +88,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 + uses: tj-actions/changed-files@v44 with: files: | deploy-config/egress_proxy/notify-admin-production.*.acl diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d74ba3133..8cf33babc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,96 +17,96 @@ jobs: environment: staging steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 + - uses: actions/checkout@v4 + with: + fetch-depth: 2 - - name: Check for changes to Terraform - id: changed-terraform-files - uses: tj-actions/changed-files@v41.0.0 - with: - files: | - terraform/staging - terraform/shared - .github/workflows/deploy.yml - - name: Terraform init - if: steps.changed-terraform-files.outputs.any_changed == 'true' - working-directory: terraform/staging - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} - run: terraform init - - name: Terraform apply - if: steps.changed-terraform-files.outputs.any_changed == 'true' - working-directory: terraform/staging - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} - TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} - TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} - run: terraform apply -auto-approve -input=false + - name: Check for changes to Terraform + id: changed-terraform-files + uses: tj-actions/changed-files@v44 + with: + files: | + terraform/staging/** + terraform/shared/** + .github/workflows/deploy.yml + - name: Terraform init + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + run: terraform init + - name: Terraform apply + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} + TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + run: terraform apply -auto-approve -input=false - - uses: ./.github/actions/setup-project + - uses: ./.github/actions/setup-project - - name: Create requirements.txt - run: poetry export --without-hashes --format=requirements.txt > requirements.txt + - name: Create requirements.txt + run: poetry export --without-hashes --format=requirements.txt > requirements.txt - - name: Deploy to cloud.gov - uses: 18f/cg-deploy-action@main - env: - DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} - SECRET_KEY: ${{ secrets.SECRET_KEY }} - ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} - NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} - NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }} - COMMIT_HASH: ${{ github.sha }} - LOGIN_PEM: ${{ secrets.LOGIN_PEM }} - LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov" - LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo" - LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token" - LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out" - LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" - LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out" - LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE" - with: - cf_username: ${{ secrets.CLOUDGOV_USERNAME }} - cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} - cf_org: gsa-tts-benefits-studio - cf_space: notify-staging - push_arguments: >- - --vars-file deploy-config/staging.yml - --var DANGEROUS_SALT="$DANGEROUS_SALT" - --var SECRET_KEY="$SECRET_KEY" - --var ADMIN_CLIENT_USERNAME="notify-admin" - --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" - --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" - --var NR_BROWSER_KEY="$NR_BROWSER_KEY" - --var COMMIT_HASH="$COMMIT_HASH" - --var LOGIN_PEM="$LOGIN_PEM" - --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID" - --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL" - --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL" - --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL" - --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" - --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" - --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" + - name: Deploy to cloud.gov + uses: 18f/cg-deploy-action@main + env: + DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }} + COMMIT_HASH: ${{ github.sha }} + LOGIN_PEM: ${{ secrets.LOGIN_PEM }} + LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov" + LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo" + LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token" + LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out" + LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" + LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out" + LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE" + with: + cf_username: ${{ secrets.CLOUDGOV_USERNAME }} + cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + cf_org: gsa-tts-benefits-studio + cf_space: notify-staging + push_arguments: >- + --vars-file deploy-config/staging.yml + --var DANGEROUS_SALT="$DANGEROUS_SALT" + --var SECRET_KEY="$SECRET_KEY" + --var ADMIN_CLIENT_USERNAME="notify-admin" + --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" + --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" + --var NR_BROWSER_KEY="$NR_BROWSER_KEY" + --var COMMIT_HASH="$COMMIT_HASH" + --var LOGIN_PEM="$LOGIN_PEM" + --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID" + --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL" + --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL" + --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL" + --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" + --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" + --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" - - name: Check for changes to egress config - id: changed-egress-config - uses: tj-actions/changed-files@v41.0.0 - with: - files: | - deploy-config/egress_proxy/notify-admin-staging.*.acl - .github/actions/deploy-proxy/action.yml - .github/workflows/deploy.yml - - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' - uses: ./.github/actions/deploy-proxy - with: - cf_space: notify-staging - app: notify-admin-staging + - name: Check for changes to egress config + id: changed-egress-config + uses: tj-actions/changed-files@v44 + with: + files: | + deploy-config/egress_proxy/notify-admin-staging.*.acl + .github/actions/deploy-proxy/action.yml + .github/workflows/deploy.yml + - name: Deploy egress proxy + if: steps.changed-egress-config.outputs.any_changed == 'true' + uses: ./.github/actions/deploy-proxy + with: + cf_space: notify-staging + app: notify-admin-staging bail: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 1c4df2aa5..15a86ae00 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ You will need the following items: [Follow the instructions here to set up the Notify.gov API.](https://github.com/GSA/notifications-api#before-you-start) The Notify.gov API is required in order for the Notify.gov Admin UI to run, and -it will also take care of many of the steps that are listed here. The sections +it will also take care of many of the steps that are listed here. The sections that are a repeat from the API setup are flagged with an **[API Step]** label in front of them. @@ -83,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these locations: For BASH shells: + - `~/.bashrc` - `~/.bash_profile` - `~/.profile` For ZSH shells: + - `~/.zshrc` - `~/.zprofile` @@ -97,7 +99,7 @@ environments. Which file you need to modify depends on whether or not you are running an interactive shell or a login shell (see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell) -for an explanation of the differences). If you're still not sure, please ask +for an explanation of the differences). If you're still not sure, please ask the team for help! Once you determine which file you'll need to modify, add these lines before any @@ -158,7 +160,7 @@ _NOTE: This project currently uses the latest `1.4.x release of Terraform._ #### [API Step] Python Installation Now we're going to install a tool to help us manage Python versions and -virtual environments on our system. First, we'll install +virtual environments on our system. First, we'll install [pyenv](https://github.com/pyenv/pyenv) and one of its plugins, [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew: @@ -285,7 +287,7 @@ we'll use `3.12` in our example here since we recently upgraded to this version: pyenv install 3.12 ``` -Next, delete the virtual environment you previously had set up. If you followed +Next, delete the virtual environment you previously had set up. If you followed the instructions above with the first-time set up, you can do this with `pyenv`: ```sh @@ -306,6 +308,20 @@ you'll be set with an upgraded version of Python. _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._ +#### Updating the .env file for Login.gov + +To configure the application for Login.gov, you will need to update the following environment variables in the .env file: + +``` +COMMIT_HASH=”--------” +``` + +Reach out to someone on the team to get the most recent Login.gov key. + +``` +LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE" +``` + #### Updating the .env file for E2E tests With the newly created `.env` file in place, you'll need to make one more @@ -353,7 +369,7 @@ API is running as well! ## Creating a 'First User' in the database After you have completed all setup steps, you will be unable to log in, because there -will not be a user in the database to link to the login.gov account you are using. So +will not be a user in the database to link to the login.gov account you are using. So you will need to create that user in your database using the 'create-test-user' command. Open two terminals pointing to the api project and then run these commands in the @@ -372,8 +388,6 @@ is the same one you are using in login.gov and make sure your phone number is in If for any reason in the course of development it is necessary for your to delete your db via the `dropdb` command, you will need to repeat these steps when you recreate your db. - - ## Git Hooks We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to diff --git a/app/config.py b/app/config.py index 77138ca16..960d6331b 100644 --- a/app/config.py +++ b/app/config.py @@ -53,7 +53,13 @@ class Config(object): PERMANENT_SESSION_LIFETIME = 1800 # 30 Minutes SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45 - ACTIVITY_STATS_LIMIT_DAYS = 7 + ACTIVITY_STATS_LIMIT_DAYS = { + "today": 0, + "one_day": 1, + "three_day": 3, + "five_day": 5, + "seven_day": 7, + } SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_NAME = "notify_admin_session" SESSION_COOKIE_SECURE = True diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index 010d19b26..42a4de090 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -143,11 +143,40 @@ def view_notifications(service_id, message_type=None): True: ["reference"], False: [], }.get(bool(current_service.api_keys)), - download_link=url_for( + download_link_one_day=url_for( ".download_notifications_csv", service_id=current_service.id, message_type=message_type, status=request.args.get("status"), + number_of_days="one_day", + ), + download_link_today=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="today", + ), + download_link_three_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="three_day", + ), + download_link_five_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="five_day", + ), + download_link_seven_day=url_for( + ".download_notifications_csv", + service_id=current_service.id, + message_type=message_type, + status=request.args.get("status"), + number_of_days="seven_day", ), ) @@ -183,10 +212,9 @@ def get_notifications(service_id, message_type, status_override=None): # noqa filter_args["status"] = set_status_filters(filter_args) service_data_retention_days = None search_term = request.form.get("to", "") - if message_type is not None: service_data_retention_days = current_service.get_days_of_retention( - message_type + message_type, number_of_days="seven_day" ) if request.path.endswith("csv") and current_user.has_permissions("view_activity"): @@ -212,7 +240,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa ) url_args = {"message_type": message_type, "status": request.args.get("status")} prev_page = None - if "links" in notifications and notifications["links"].get("prev", None): prev_page = generate_previous_dict( "main.view_notifications", service_id, page, url_args=url_args @@ -233,7 +260,6 @@ def get_notifications(service_id, message_type, status_override=None): # noqa ) else: download_link = None - return { "service_data_retention_days": service_data_retention_days, "counts": render_template( @@ -362,6 +388,7 @@ def get_job_partials(job): filter_args = parse_filter_args(request.args) filter_args["status"] = set_status_filters(filter_args) notifications = job.get_notifications(status=filter_args["status"]) + number_of_days = "seven_day" counts = render_template( "partials/count.html", counts=_get_job_counts(job), @@ -371,7 +398,7 @@ def get_job_partials(job): ), ) service_data_retention_days = current_service.get_days_of_retention( - job.template_type + job.template_type, number_of_days ) if request.referrer is not None: diff --git a/app/main/views/notifications.py b/app/main/views/notifications.py index ac05e05ff..e41708b8c 100644 --- a/app/main/views/notifications.py +++ b/app/main/views/notifications.py @@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification): def download_notifications_csv(service_id): filter_args = parse_filter_args(request.args) filter_args["status"] = set_status_filters(filter_args) - + number_of_days = request.args["number_of_days"] service_data_retention_days = current_service.get_days_of_retention( - filter_args.get("message_type")[0] + filter_args.get("message_type")[0], number_of_days ) file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p") file_time = f"{file_time} {get_user_preferred_timezone()}" diff --git a/app/main/views/send.py b/app/main/views/send.py index 8bb6ce24c..bb63f10a9 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -51,7 +51,6 @@ from app.utils.templates import get_template from app.utils.user import user_has_permissions from notifications_utils import SMS_CHAR_COUNT_LIMIT from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.logging import scrub from notifications_utils.recipients import RecipientCSV, first_column_headings from notifications_utils.sanitise_text import SanitiseASCII @@ -953,9 +952,6 @@ def send_notification(service_id, template_id): ) ) - current_app.logger.info( - hilite(scrub(f"Recipient for the one-off will be {recipient}")) - ) keys = [] values = [] for k, v in session["placeholders"].items(): @@ -971,6 +967,12 @@ def send_notification(service_id, template_id): ) my_data = {"filename": filename, "template_id": template_id, "data": data} upload_id = s3upload(service_id, my_data) + + # To debug messages that the user reports have not been sent, we log + # the csv filename and the job id. The user will give us the file name, + # so we can search on that to obtain the job id, which we can use elsewhere + # on the API side to find out what happens to the message. + current_app.logger.info(hilite(f"One-off file: {filename} job_id: {upload_id}")) form = CsvUploadForm() form.file.data = my_data form.file.name = filename @@ -989,19 +991,6 @@ def send_notification(service_id, template_id): valid="True", ) - # Here we are attempting to cleverly link the job id to the one-off recipient - # If we know the partial phone number of the recipient, we can search - # on that initially and find this, which will give us the job_id - # And once we know the job_id, we can search on that and it might tell us something - # about report generation. - current_app.logger.info( - hilite( - scrub( - f"Created job to send one-off, recipient is {recipient}, job_id is {upload_id}" - ) - ) - ) - session.pop("recipient") session.pop("placeholders") @@ -1033,7 +1022,12 @@ def send_notification(service_id, template_id): job_id=upload_id, ) ) - + total = notifications["total"] + current_app.logger.info( + hilite( + f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}" + ) + ) return redirect( url_for( ".view_job", diff --git a/app/models/service.py b/app/models/service.py index 375047d8d..e9bcf8a7d 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin): def get_data_retention_item(self, id): return next((dr for dr in self.data_retention if dr["id"] == id), None) - def get_days_of_retention(self, notification_type): + def get_days_of_retention(self, notification_type, number_of_days): return next( ( dr @@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin): if dr["notification_type"] == notification_type ), {}, - ).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"]) + ).get( + "days_of_retention", + current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days), + ) @cached_property def organization(self): diff --git a/app/templates/views/notifications.html b/app/templates/views/notifications.html index 9f5accd43..cac2b9811 100644 --- a/app/templates/views/notifications.html +++ b/app/templates/views/notifications.html @@ -64,10 +64,22 @@ {% if current_user.has_permissions('view_activity') %}

- Download this report (CSV) + Download all data last 7 days (CSV)   Data available for {{ partials.service_data_retention_days }} days

+

+ Download all data last 5 days (CSV) +   +

+

+ Download all data last 3 days (CSV) +   +

+

+ Download all data today (CSV) +   +

{% endif %} {{ ajax_block( diff --git a/app/utils/csv.py b/app/utils/csv.py index 5c5b794de..4ed6d16b5 100644 --- a/app/utils/csv.py +++ b/app/utils/csv.py @@ -7,7 +7,6 @@ from flask_login import current_user from app.models.spreadsheet import Spreadsheet from app.utils import hilite from app.utils.templates import get_sample_template -from notifications_utils.logging import scrub from notifications_utils.recipients import RecipientCSV @@ -74,12 +73,11 @@ def generate_notifications_csv(**kwargs): # This generates the "batch" csv report if kwargs.get("job_id"): - # The kwargs contain the job id, which is linked to the recipient's partial phone number in other debug # Some unit tests are mocking the kwargs and turning them into a function instead of dict, # hence the try/except. try: current_app.logger.info( - hilite(f"Setting up report with kwargs {scrub(json.dumps(kwargs))}") + hilite(f"Setting up report with kwargs {json.dumps(kwargs)}") ) except TypeError: pass @@ -89,7 +87,7 @@ def generate_notifications_csv(**kwargs): # we display to 999 characters, because we don't want to show the contents for reports with thousands of rows. current_app.logger.info( hilite( - f"Original csv for job_id {kwargs['job_id']}: {scrub(original_file_contents[0:999])}" + f"Original csv for job_id {kwargs['job_id']}: {original_file_contents[0:999]}" ) ) original_upload = RecipientCSV( diff --git a/deploy-config/production.yml b/deploy-config/production.yml index 18b5cfdcc..b38094f28 100644 --- a/deploy-config/production.yml +++ b/deploy-config/production.yml @@ -1,6 +1,6 @@ env: production instances: 2 -memory: 1G +memory: 2G public_admin_route: beta.notify.gov cloud_dot_gov_route: notify.app.cloud.gov redis_enabled: 1 diff --git a/gunicorn_config.py b/gunicorn_config.py index 4f928a467..f9eb4f17d 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -5,7 +5,10 @@ import multiprocessing import gunicorn # Let gunicorn figure out the right number of workers -workers = multiprocessing.cpu_count() * 2 + 1 +# The recommended formula is cpu_count() * 2 + 1 +# but we have an unusual configuration with a lot of cpus and not much memory +# so adjust it. +workers = multiprocessing.cpu_count() worker_class = "eventlet" bind = "0.0.0.0:{}".format(os.getenv("PORT")) disable_redirect_access_to_syslog = True @@ -16,23 +19,3 @@ def worker_abort(worker): worker.log.info("worker received ABORT") for stack in sys._current_frames().values(): worker.log.error("".join(traceback.format_stack(stack))) - - -# This issue is fixed in the 22.0.0 release, which we are using -# See github issue for details -# def fix_ssl_monkeypatching(): -# """ -# eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently -# a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However -# eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not -# ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type -# :scream: -# https://github.com/eventlet/eventlet/issues/692 -# """ -# # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again. -# # eventlet.monkey_patch can be called multiple times without issue -# eventlet.monkey_patch() -# eventlet.green.ssl.timeout_exc = socket.timeout - - -# fix_ssl_monkeypatching() diff --git a/manifest.yml b/manifest.yml index ed596ffeb..bccd00db6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -12,7 +12,7 @@ applications: - route: ((cloud_dot_gov_route)) services: - - notify-admin-redis-((env)) + - notify-admin-redis-v70-((env)) - notify-api-csv-upload-bucket-((env)) - notify-admin-logo-upload-bucket-((env)) diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 7c56a00ad..4abe5bcf7 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -70,6 +70,7 @@ def configure_handler(handler, app, formatter): handler.addFilter(AppNameFilter(app.config["NOTIFY_APP_NAME"])) handler.addFilter(RequestIdFilter()) handler.addFilter(ServiceIdFilter()) + handler.addFilter(PIIFilter()) return handler @@ -134,13 +135,30 @@ class JSONFormatter(BaseJSONFormatter): return log_record -def scrub(msg): - # Eventually we want to scrub all messages in all logs for phone numbers - # and email addresses, masking them. Ultimately this will probably get - # refactored into a 'SafeLogger' subclass or something, but let's start here - # with phones. - phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) - phones = [phone.replace("-", "").replace(" ", "") for phone in phones] - for phone in phones: - msg = msg.replace(phone, f"1XXXXX{phone[-5:]}") - return msg +class PIIFilter(logging.Filter): + def scrub(self, msg): + # Eventually we want to scrub all messages in all logs for phone numbers + # and email addresses, masking them. Ultimately this will probably get + # refactored into a 'SafeLogger' subclass or something, but let's start here + # with phones. + + # Sometimes just an exception object is passed in for the message, skip those. + if not isinstance(msg, str): + return msg + phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg) + phones = [phone.replace("-", "").replace(" ", "") for phone in phones] + for phone in phones: + msg = msg.replace(phone, "1XXXXXXXXXX") + + emails = re.findall( + r"[\w\.-]+@[\w\.-]+", msg + ) # ['alice@google.com', 'bob@abc.com'] + for email in emails: + # do something with each found email string + masked_email = "XXXXX@XXXXXXX" + msg = msg.replace(email, masked_email) + return msg + + def filter(self, record): + record.msg = self.scrub(record.msg) + return record diff --git a/terraform/bootstrap/providers.tf b/terraform/bootstrap/providers.tf index b6f27acf8..cce97ee8f 100644 --- a/terraform/bootstrap/providers.tf +++ b/terraform/bootstrap/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf index 7f34fb0f1..545871d4a 100644 --- a/terraform/demo/main.tf +++ b/terraform/demo/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = false } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf index 2ced7915f..2381dcd28 100644 --- a/terraform/demo/providers.tf +++ b/terraform/demo/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/development/providers.tf b/terraform/development/providers.tf index 5dcaece3e..7b9ce5c7f 100644 --- a/terraform/development/providers.tf +++ b/terraform/development/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 450212cdf..69cc9b264 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = false } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-3node-large" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-3node-large" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf index ed822db44..97f543a23 100644 --- a/terraform/production/providers.tf +++ b/terraform/production/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index 74c16d808..b831b44d6 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -6,7 +6,7 @@ locals { recursive_delete = true } -module "redis" { +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +16,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf index 09911edc4..978b10f45 100644 --- a/terraform/sandbox/providers.tf +++ b/terraform/sandbox/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/terraform/shared/container_networking/providers.tf b/terraform/shared/container_networking/providers.tf index 21ac567a2..dec8379ee 100644 --- a/terraform/shared/container_networking/providers.tf +++ b/terraform/shared/container_networking/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } } diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index d0df6e81d..0cc72358a 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -6,7 +6,15 @@ locals { recursive_delete = true } -module "redis" { +resource "null_resource" "prevent_destroy" { + + lifecycle { + prevent_destroy = false # destroying staging is allowed + } +} + + +module "redis" { # default v6.2; delete after v7.0 resource is bound source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1" cf_org_name = local.cf_org_name @@ -16,6 +24,20 @@ module "redis" { redis_plan_name = "redis-dev" } +module "redis-v70" { + source = "github.com/GSA-TTS/terraform-cloudgov//redis?ref=v1.0.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + name = "${local.app_name}-redis-v70-${local.env}" + redis_plan_name = "redis-dev" + json_params = jsonencode( + { + "engineVersion" : "7.0", + } + ) +} + module "logo_upload_bucket" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1" diff --git a/terraform/staging/providers.tf b/terraform/staging/providers.tf index d6928e61e..05d8b90d3 100644 --- a/terraform/staging/providers.tf +++ b/terraform/staging/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = "~> 1.0" + required_version = "~> 1.7" required_providers { cloudfoundry = { source = "cloudfoundry-community/cloudfoundry" - version = "0.53.0" + version = "0.53.1" } } diff --git a/tests/app/main/views/test_activity.py b/tests/app/main/views/test_activity.py index 3493aaae4..3bbe40282 100644 --- a/tests/app/main/views/test_activity.py +++ b/tests/app/main/views/test_activity.py @@ -228,12 +228,18 @@ def test_can_show_notifications_if_data_retention_not_available( url_for, ".download_notifications_csv", message_type=None, + number_of_days="seven_day", ), ), ( create_active_user_with_permissions(), {"status": "failed"}, - partial(url_for, ".download_notifications_csv", status="failed"), + partial( + url_for, + ".download_notifications_csv", + status="failed", + number_of_days="seven_day", + ), ), ( create_active_user_with_permissions(), @@ -242,15 +248,13 @@ def test_can_show_notifications_if_data_retention_not_available( url_for, ".download_notifications_csv", message_type="sms", + number_of_days="seven_day", ), ), ( create_active_user_view_permissions(), {}, - partial( - url_for, - ".download_notifications_csv", - ), + partial(url_for, ".download_notifications_csv", number_of_days="seven_day"), ), ( create_active_caseworking_user(), diff --git a/tests/notifications_utils/test_logging.py b/tests/notifications_utils/test_logging.py index 858b9352b..767c1f6f8 100644 --- a/tests/notifications_utils/test_logging.py +++ b/tests/notifications_utils/test_logging.py @@ -51,11 +51,16 @@ def test_base_json_formatter_contains_service_id(): assert service_id_filter.filter(record).service_id == "no-service-id" -def test_scrub(): - result = logging.scrub( - "This is a message with 17775554324, and also 18884449323 and also 17775554324" - ) - assert ( - result - == "This is a message with 1XXXXX54324, and also 1XXXXX49323 and also 1XXXXX54324" +def test_pii_filter(): + record = builtin_logging.LogRecord( + name="log thing", + level="info", + pathname="path", + lineno=123, + msg="phone1: 1555555555, phone2: 1555555554, email1: fake@fake.gov, email2: fake@fake2.fake.gov", + exc_info=None, + args=None, ) + pii_filter = logging.PIIFilter() + clean_msg = "phone1: 1XXXXXXXXXX, phone2: 1XXXXXXXXXX, email1: XXXXX@XXXXXXX, email2: XXXXX@XXXXXXX" + assert pii_filter.filter(record).msg == clean_msg