From 37de8f352e6f892c7a8385ee7b5d73633e4d686a Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 18 Sep 2023 16:49:45 -0400 Subject: [PATCH 01/13] add docs about logs & env vars --- docs/all.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/all.md b/docs/all.md index 10134db31..7d2b7e432 100644 --- a/docs/all.md +++ b/docs/all.md @@ -5,12 +5,14 @@ - [New Relic](#new-relic) - [Onboarding](#onboarding) - [Setting up the infrastructure](#setting-up-the-infrastructure) +- [Using the logs](#using-the-logs) - [Testing](#testing) - [CI testing](#ci-testing) - [Manual testing](#manual-testing) - [To run a local OWASP scan](#to-run-a-local-owasp-scan) - [Deploying](#deploying) - [Egress Proxy](#egress-proxy) + - [Managing environment variables](#managing-environment-variables) - [Sandbox environment](#sandbox-environment) - [Database management](#database-management) - [Initial state](#initial-state) @@ -205,6 +207,12 @@ Example answers for toll-free registration form ![example answers for toll-free registration form](./toll-free-registration.png) +# Using the logs + +If you're using the `cf` CLI, you can run `cf logs notify-api-ENV` and/or `cf logs notify-admin-ENV` to stream logs in real time. Add `--recent` to get the last few logs, though logs often move pretty quickly. + +For general log searching, [the cloud.gov Kibana](https://logs.fr.cloud.gov/) is powerful, though quite complex to get started. For shortcuts to errors, some team members have New Relic access. + # Testing ``` @@ -304,6 +312,26 @@ application to a select list of allowed domains. Update the allowed domains by updating `deploy-config/egress_proxy/notify-api-.allow.acl` and deploying an updated version of the application throught he normal deploy process. +## Managing environment variables + +For an environment variable to make its way into the cloud.gov environment, it *must* end up in the `manifest.yml` file. Based on the deployment approach described above, there are 2 ways for this to happen. + +### Secret environment variables + +Because secrets are pulled from GitHub, they must be passed from our action to the deploy action and then placed into `manifest.yml`. This means that they should be in a 4 places: + +- [ ] The GitHub secrets store +- [ ] The deploy action in the `env` section using the format `{secrets.SECRET_NAME}` +- [ ] The deploy action in the `push_arguments` section using the format `--var SECRET_NAME="$SECRET_NAME"` +- [ ] The manifest using the format `((SECRET_NAME))` + +### Public environment variables + +Public env vars make up the configuration in `deploy-config`. These are pulled in together by the `--vars-file` line in the deploy action. To add or update one, it should be in 2 places: + +- [ ] The relevant YAML file in `deploy-config` using the format `var_name: value` +- [ ] The manifest using the format `((var_name))` + ## Sandbox environment There is a sandbox space, complete with terraform and `deploy-config/sandbox.yml` file available From 04bd91758b87ec762ea560a30dd5779d6e1f9ea0 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 18 Sep 2023 16:50:52 -0400 Subject: [PATCH 02/13] update doc links in readme --- README.md | 96 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index eb68b03f8..35396711f 100644 --- a/README.md +++ b/README.md @@ -100,53 +100,55 @@ A direct installation of PostgreSQL will not put the `createdb` command on your ## Documentation -- [Infrastructure overview](#infrastructure-overview) - - [GitHub Repositories](#github-repositories) - - [Terraform](#terraform) - - [AWS](#aws) - - [New Relic](#new-relic) - - [Onboarding](#onboarding) - - [Setting up the infrastructure](#setting-up-the-infrastructure) -- [Testing](#testing) - - [CI testing](#ci-testing) - - [Manual testing](#manual-testing) - - [To run a local OWASP scan](#to-run-a-local-owasp-scan) -- [Deploying](#deploying) - - [Egress Proxy](#egress-proxy) - - [Sandbox environment](#sandbox-environment) -- [Database management](#database-management) - - [Initial state](#initial-state) - - [Data Model Diagram](#data-model-diagram) - - [Migrations](#migrations) - - [Purging user data](#purging-user-data) -- [One-off tasks](#one-off-tasks) -- [How messages are queued and sent](#how-messages-are-queued-and-sent) -- [Writing public APIs](#writing-public-apis) - - [Overview](#overview) - - [Documenting APIs](#documenting-apis) - - [New APIs](#new-apis) -- [API Usage](#api-usage) - - [Connecting to the API](#connecting-to-the-api) - - [Postman Documentation](#postman-documentation) - - [Using OpenAPI documentation](#using-openapi-documentation) -- [Queues and tasks](#queues-and-tasks) - - [Priority queue](#priority-queue) - - [Celery scheduled tasks](#celery-scheduled-tasks) -- [US Notify](#us-notify) - - [System Description](#system-description) -- [Run Book](#run-book) - - [ Alerts, Notifications, Monitoring](#-alerts-notifications-monitoring) - - [ Restaging Apps](#-restaging-apps) - - [ Smoke-testing the App](#-smoke-testing-the-app) - - [ Configuration Management](#-configuration-management) - - [ DNS Changes](#-dns-changes) - - [Exporting test results for compliance monitoring](#exporting-test-results-for-compliance-monitoring) - - [ Known Gotchas](#-known-gotchas) - - [ User Account Management](#-user-account-management) - - [ SMS Phone Number Management](#-sms-phone-number-management) -- [Data Storage Policies \& Procedures](#data-storage-policies--procedures) - - [Potential PII Locations](#potential-pii-locations) - - [Data Retention Policy](#data-retention-policy) +- [Infrastructure overview](./docs/all.md#infrastructure-overview) + - [GitHub Repositories](./docs/all.md#github-repositories) + - [Terraform](./docs/all.md#terraform) + - [AWS](./docs/all.md#aws) + - [New Relic](./docs/all.md#new-relic) + - [Onboarding](./docs/all.md#onboarding) + - [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure) +- [Using the logs](./docs/all.md#using-the-logs) +- [Testing](./docs/all.md#testing) + - [CI testing](./docs/all.md#ci-testing) + - [Manual testing](./docs/all.md#manual-testing) + - [To run a local OWASP scan](./docs/all.md#to-run-a-local-owasp-scan) +- [Deploying](./docs/all.md#deploying) + - [Egress Proxy](./docs/all.md#egress-proxy) + - [Managing environment variables](./docs/all.md#managing-environment-variables) + - [Sandbox environment](./docs/all.md#sandbox-environment) +- [Database management](./docs/all.md#database-management) + - [Initial state](./docs/all.md#initial-state) + - [Data Model Diagram](./docs/all.md#data-model-diagram) + - [Migrations](./docs/all.md#migrations) + - [Purging user data](./docs/all.md#purging-user-data) +- [One-off tasks](./docs/all.md#one-off-tasks) +- [How messages are queued and sent](./docs/all.md#how-messages-are-queued-and-sent) +- [Writing public APIs](./docs/all.md#writing-public-apis) + - [Overview](./docs/all.md#overview) + - [Documenting APIs](./docs/all.md#documenting-apis) + - [New APIs](./docs/all.md#new-apis) +- [API Usage](./docs/all.md#api-usage) + - [Connecting to the API](./docs/all.md#connecting-to-the-api) + - [Postman Documentation](./docs/all.md#postman-documentation) + - [Using OpenAPI documentation](./docs/all.md#using-openapi-documentation) +- [Queues and tasks](./docs/all.md#queues-and-tasks) + - [Priority queue](./docs/all.md#priority-queue) + - [Celery scheduled tasks](./docs/all.md#celery-scheduled-tasks) +- [US Notify](./docs/all.md#us-notify) + - [System Description](./docs/all.md#system-description) +- [Run Book](./docs/all.md#run-book) + - [ Alerts, Notifications, Monitoring](./docs/all.md#-alerts-notifications-monitoring) + - [ Restaging Apps](./docs/all.md#-restaging-apps) + - [ Smoke-testing the App](./docs/all.md#-smoke-testing-the-app) + - [ Configuration Management](./docs/all.md#-configuration-management) + - [ DNS Changes](./docs/all.md#-dns-changes) + - [Exporting test results for compliance monitoring](./docs/all.md#exporting-test-results-for-compliance-monitoring) + - [ Known Gotchas](./docs/all.md#-known-gotchas) + - [ User Account Management](./docs/all.md#-user-account-management) + - [ SMS Phone Number Management](./docs/all.md#-sms-phone-number-management) +- [Data Storage Policies \& Procedures](./docs/all.md#data-storage-policies--procedures) + - [Potential PII Locations](./docs/all.md#potential-pii-locations) + - [Data Retention Policy](./docs/all.md#data-retention-policy) ## License && public domain From 8921db66161aa2cfef29f2059da13b2e77a02e3f Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 21 Sep 2023 13:27:53 -0400 Subject: [PATCH 03/13] some terraform context --- docs/all.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/all.md b/docs/all.md index 7d2b7e432..6a9968a22 100644 --- a/docs/all.md +++ b/docs/all.md @@ -87,13 +87,17 @@ In addition to terraform directories in the api and admin apps above: ## Terraform +We use Terraform to manage our infrastructure, providing consistent setups across the environments. + +Our Terraform configurations manage components via cloud.gov. This means that the configurations should work out of the box if you are using a Cloud Foundry platform, but will not work for setups based on raw AWS. + ### Development There are several remote services required for local development: -* s3 -* ses -* sns +* S3 +* SES +* SNS Credentials for these services are created by running: From 141deb3880d3ce35f9824f989973f400646da273 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 21 Sep 2023 13:50:36 -0700 Subject: [PATCH 04/13] notify-admin-768 initial --- app/celery/tasks.py | 8 ++++++++ migrations/versions/0402_total_message_limit_default.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 31aee0dd6..89211ef43 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -31,6 +31,7 @@ from app.models import ( ) from app.notifications.process_notifications import persist_notification from app.notifications.validators import check_service_over_total_message_limit +from app.schema_validation.definitions import uuid from app.serialised_models import SerialisedService, SerialisedTemplate from app.service.utils import service_allowed_to_send_to from app.utils import DATETIME_FORMAT @@ -193,6 +194,12 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i return try: + job_id = notification.get("job", None) + created_by_id = None + if job_id: + job = dao_get_job_by_id(job_id) + created_by_id = job.created_by_id + saved_notification = persist_notification( template_id=notification["template"], template_version=notification["template_version"], @@ -203,6 +210,7 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i api_key_id=None, key_type=KEY_TYPE_NORMAL, created_at=datetime.utcnow(), + created_by_id=uuid.UUID(created_by_id), job_id=notification.get("job", None), job_row_number=notification.get("row_number", None), notification_id=notification_id, diff --git a/migrations/versions/0402_total_message_limit_default.py b/migrations/versions/0402_total_message_limit_default.py index 786e60362..75539bcf1 100644 --- a/migrations/versions/0402_total_message_limit_default.py +++ b/migrations/versions/0402_total_message_limit_default.py @@ -10,8 +10,8 @@ from flask import current_app import sqlalchemy as sa -revision = '0402_total_message_limit_default' -down_revision = '0401_add_e2e_test_user' +revision = "0402_total_message_limit_default" +down_revision = "0401_add_e2e_test_user" def upgrade(): From 308dbf16a8e6fa637d6a4e5bb4c9a32e84f9cbba Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 22 Sep 2023 12:49:22 -0400 Subject: [PATCH 05/13] use our own adr action to try things out --- .github/workflows/adr-accepted.yml | 81 +++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/.github/workflows/adr-accepted.yml b/.github/workflows/adr-accepted.yml index bfe151ff4..7adfab007 100644 --- a/.github/workflows/adr-accepted.yml +++ b/.github/workflows/adr-accepted.yml @@ -1,18 +1,83 @@ name: ADR accepted + on: issues: types: - closed jobs: - main: - name: ADR accepted + accept: runs-on: ubuntu-latest - steps: - - name: memorialize the ADR - uses: 18F/adr-automation/accepted@actioning + + - name: check for tags + if: "${{ !contains(github.event.issue.labels.*.name, 'ADR: accepted' )}}" + shell: bash + run: exit 0 + + - name: checkout main branch + uses: actions/checkout@v3 + with: + ref: main with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - label: "ADR: accepted" - path: docs/adrs + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: get ADR number + id: next + shell: bash + run: | + mkdir -p docs/adrs + LAST_ADR=$(ls docs/adrs/*.md | grep -Eo "/[0-9]+-" | sort | tail -n1 | grep -Eo "[0-9]+") + LAST_ADR=$(echo "$LAST_ADR" | sed -E 's/^0+//') + NEXT_ADR=$(($LAST_ADR + 1)) + NEXT_ADR=$(printf "%04i" "$NEXT_ADR") + echo "number=$NEXT_ADR" >> "$GITHUB_OUTPUT" + + - name: get date + id: date + shell: bash + run: echo "date=$(date +'%B %d, %Y')" >> "$GITHUB_OUTPUT" + + - name: build filename + id: filename + shell: bash + run: | + SLUG=$(printf '%q\n' "${{ github.event.issue.title }}" | tr A-Z a-z) + SLUG=$(printf '%q\n' "$SLUG" | iconv -c -t ascii//TRANSLIT) + SLUG=$(printf '%q\n' "$SLUG" | sed -E 's/[^a-z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/^-+|-+$//g') + + FILENAME="docs/adrs/${{ steps.next.outputs.number }}-$SLUG.md" + echo "slug=$SLUG" >> "$GITHUB_OUTPUT" + echo "filename=$FILENAME" >> "$GITHUB_OUTPUT" + + - name: write the ADR + uses: DamianReeves/write-file-action@v1.2 + with: + path: ${{ steps.filename.outputs.filename }} + write-mode: overwrite + contents: | + # ${{ github.event.issue.title }} + + Status: Accepted + Date: ${{ steps.date.outputs.date }} + + ${{ github.event.issue.body }} + + - name: branch, commit, and open PR + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BRANCH="adr/auto/${{ steps.filename.outputs.slug }}" + git config --global user.email "tts@gsa.gov" + git config --global user.name "Notify ADR Automation" + git checkout -b $BRANCH + git add docs/adrs/*.md + git commit -m "add ADR ${{ steps.next.outputs.number }}: ${{ github.event.issue.title }}" + git push -f origin $BRANCH + gh pr create \ + --title "Add ADR ${{ steps.next.outputs.number }} to the repo" \ + --body "This pull request was opened automatically because #${{ github.event.issue.number }} was closed after being marked as an approved ADR. It contains a markdown file capturing the ADR body at the time the issue was closed. Please verify that the markdown is correct before merging!" || true + gh pr merge $BRANCH --auto --squash || true + + \ No newline at end of file From f80ae2d0b16838919b33dfa4fcf3413d779e5779 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 22 Sep 2023 16:07:36 -0400 Subject: [PATCH 06/13] add links to kibana filtered views --- docs/all.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/all.md b/docs/all.md index 6a9968a22..268d6ba89 100644 --- a/docs/all.md +++ b/docs/all.md @@ -217,6 +217,14 @@ If you're using the `cf` CLI, you can run `cf logs notify-api-ENV` and/or `cf lo For general log searching, [the cloud.gov Kibana](https://logs.fr.cloud.gov/) is powerful, though quite complex to get started. For shortcuts to errors, some team members have New Relic access. +The links below will open a filtered view with logs from both applications, which can then be filtered further. However, for the links to work, you need to paste them into the URL bar while *already* on the Kibana page. If not, you'll just be redirected to the generic dashboard. + +Production: https://logs.fr.cloud.gov/app/discover#/view/218a6790-596d-11ee-a43a-090d426b9a38 +Demo: https://logs.fr.cloud.gov/app/discover#/view/891392a0-596e-11ee-921a-1b6b2f4d89ed +Staging: https://logs.fr.cloud.gov/app/discover#/view/73d7c820-596e-11ee-a43a-090d426b9a38 + +Once in the view, you'll likely want to adjust the time range in the upper right of the page. + # Testing ``` From b61ca002c3778d01d79807465da07b6a451fe670 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 22 Sep 2023 17:03:04 -0400 Subject: [PATCH 07/13] add workflow permission --- .github/workflows/adr-accepted.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/adr-accepted.yml b/.github/workflows/adr-accepted.yml index 7adfab007..84e9be66e 100644 --- a/.github/workflows/adr-accepted.yml +++ b/.github/workflows/adr-accepted.yml @@ -5,6 +5,9 @@ on: types: - closed +permissions: + contents: read + jobs: accept: runs-on: ubuntu-latest From f6889efa0ae5c0f3eb01fd8656d81bb7ccb93148 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Sep 2023 21:57:11 +0000 Subject: [PATCH 08/13] Bump black from 23.7.0 to 23.9.1 Bumps [black](https://github.com/psf/black) from 23.7.0 to 23.9.1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.7.0...23.9.1) --- updated-dependencies: - dependency-name: black dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index ac0df2c2a..4886eba41 100644 --- a/poetry.lock +++ b/poetry.lock @@ -309,33 +309,33 @@ files = [ [[package]] name = "black" -version = "23.7.0" +version = "23.9.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-23.7.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be"}, - {file = "black-23.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc"}, - {file = "black-23.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a"}, - {file = "black-23.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926"}, - {file = "black-23.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6"}, - {file = "black-23.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a"}, - {file = "black-23.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087"}, - {file = "black-23.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91"}, - {file = "black-23.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491"}, - {file = "black-23.7.0-py3-none-any.whl", hash = "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96"}, - {file = "black-23.7.0.tar.gz", hash = "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"}, + {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"}, + {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"}, + {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"}, + {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"}, + {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"}, + {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"}, + {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"}, + {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"}, + {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"}, + {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"}, ] [package.dependencies] @@ -345,7 +345,7 @@ packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -4291,4 +4291,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "642319df54dfc549b650ff145855594e79d21c6191af06c85632595feef13482" +content-hash = "15aafd504a605f6621f3017ddfba898373885f90926eb16eef363ba5b33b53a6" diff --git a/pyproject.toml b/pyproject.toml index 4f088b097..1e91acf7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ asn1crypto = "==1.5.1" async-timeout = "==4.0.2" attrs = "==23.1.0" awscli = "==1.29.15" -black = "==23.7.0" +black = "==23.9.1" bcrypt = "==3.2.2" beautifulsoup4 = "==4.12.2" billiard = "==3.6.4.0" From f5a486be7868d69c22c82cad3e9b0e6a00cea89f Mon Sep 17 00:00:00 2001 From: Steven Reilly Date: Fri, 22 Sep 2023 21:59:09 -0400 Subject: [PATCH 09/13] Update docs/all.md Co-authored-by: Carlo Costino --- docs/all.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/all.md b/docs/all.md index 268d6ba89..4e7e0aa8d 100644 --- a/docs/all.md +++ b/docs/all.md @@ -215,7 +215,7 @@ Example answers for toll-free registration form If you're using the `cf` CLI, you can run `cf logs notify-api-ENV` and/or `cf logs notify-admin-ENV` to stream logs in real time. Add `--recent` to get the last few logs, though logs often move pretty quickly. -For general log searching, [the cloud.gov Kibana](https://logs.fr.cloud.gov/) is powerful, though quite complex to get started. For shortcuts to errors, some team members have New Relic access. +For general log searching, [the cloud.gov Kibana instance](https://logs.fr.cloud.gov/) is powerful, though quite complex to get started. For shortcuts to errors, some team members have New Relic access. The links below will open a filtered view with logs from both applications, which can then be filtered further. However, for the links to work, you need to paste them into the URL bar while *already* on the Kibana page. If not, you'll just be redirected to the generic dashboard. From ec48d436d6bcabb727351cde50dc5474113fba97 Mon Sep 17 00:00:00 2001 From: Steven Reilly Date: Fri, 22 Sep 2023 21:59:24 -0400 Subject: [PATCH 10/13] Update docs/all.md Co-authored-by: Carlo Costino --- docs/all.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/all.md b/docs/all.md index 4e7e0aa8d..2e922f6da 100644 --- a/docs/all.md +++ b/docs/all.md @@ -217,7 +217,7 @@ If you're using the `cf` CLI, you can run `cf logs notify-api-ENV` and/or `cf lo For general log searching, [the cloud.gov Kibana instance](https://logs.fr.cloud.gov/) is powerful, though quite complex to get started. For shortcuts to errors, some team members have New Relic access. -The links below will open a filtered view with logs from both applications, which can then be filtered further. However, for the links to work, you need to paste them into the URL bar while *already* on the Kibana page. If not, you'll just be redirected to the generic dashboard. +The links below will open a filtered view with logs from both applications, which can then be filtered further. However, for the links to work, you need to paste them into the URL bar while *already* logged into and viewing the Kibana page. If not, you'll just be redirected to the generic dashboard. Production: https://logs.fr.cloud.gov/app/discover#/view/218a6790-596d-11ee-a43a-090d426b9a38 Demo: https://logs.fr.cloud.gov/app/discover#/view/891392a0-596e-11ee-921a-1b6b2f4d89ed From 6c6762b2cec7abf600d8bda140e1659e8fce68e6 Mon Sep 17 00:00:00 2001 From: Steven Reilly Date: Fri, 22 Sep 2023 21:59:47 -0400 Subject: [PATCH 11/13] Update docs/all.md Co-authored-by: Carlo Costino --- docs/all.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/all.md b/docs/all.md index 2e922f6da..276fa649d 100644 --- a/docs/all.md +++ b/docs/all.md @@ -335,7 +335,7 @@ Because secrets are pulled from GitHub, they must be passed from our action to t - [ ] The GitHub secrets store - [ ] The deploy action in the `env` section using the format `{secrets.SECRET_NAME}` - [ ] The deploy action in the `push_arguments` section using the format `--var SECRET_NAME="$SECRET_NAME"` -- [ ] The manifest using the format `((SECRET_NAME))` +- [ ] The manifest using the format `SECRET_NAME: ((SECRET_NAME))` ### Public environment variables From 6070c97ad00b696841ebfd924eb56cb8226a0e07 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 22 Sep 2023 22:15:53 -0400 Subject: [PATCH 12/13] have valid yaml instead of invalid yaml --- .github/workflows/adr-accepted.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/adr-accepted.yml b/.github/workflows/adr-accepted.yml index 84e9be66e..4f933e5e6 100644 --- a/.github/workflows/adr-accepted.yml +++ b/.github/workflows/adr-accepted.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: check for tags + - name: check for tags if: "${{ !contains(github.event.issue.labels.*.name, 'ADR: accepted' )}}" shell: bash run: exit 0 From 5e2d65a348ed060a7b499f3d1eae9c7a3b0ad281 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 25 Sep 2023 13:22:21 -0700 Subject: [PATCH 13/13] add debug for check sms deliver receipt bug --- app/clients/cloudwatch/aws_cloudwatch.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index f9eef87d4..3447cbc20 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -4,6 +4,7 @@ import re import time from boto3 import client +from flask import current_app from app.clients import AWS_CLIENT_CONFIG, Client from app.cloudfoundry_config import cloud_config @@ -50,6 +51,7 @@ class AwsCloudwatchClient(Client): # Check all cloudwatch logs from the time the notification was sent (currently 5 minutes previously) until now now = round(time.time() * 1000) beginning = sent_at + current_app.logger.info(f"TIME RANGE TO CHECK {beginning} to {now}") next_token = None all_log_events = [] while True: @@ -72,13 +74,20 @@ class AwsCloudwatchClient(Client): all_log_events.extend(log_events) if len(log_events) > 0: # We found it + current_app.logger.info( + f"WE FOUND THE EVENT WE WERE LOOKING FOR? {log_events}" + ) break next_token = response.get("nextToken") if not next_token: break + if len(all_log_events) == 0: + print(f"WE FOUND NO LOG EVENTS OVER TIME RANGE {beginning} to {now}") return all_log_events def check_sms(self, message_id, notification_id, created_at): + if os.getenv("LOCALSTACK_ENDPOINT_URL"): + current_app.logger.info("GADZOOKS WE ARE RUNNING WITH LOCALSTACK") region = cloud_config.sns_region # TODO this clumsy approach to getting the account number will be fixed as part of notify-api #258 account_number = cloud_config.ses_domain_arn @@ -87,24 +96,39 @@ class AwsCloudwatchClient(Client): account_number = account_number[0] log_group_name = f"sns/{region}/{account_number}/DirectPublishToPhoneNumber" + current_app.logger.info( + f"LOG GROUP NAME: {log_group_name} MESSAGE ID: {message_id}" + ) filter_pattern = '{$.notification.messageId="XXXXX"}' filter_pattern = filter_pattern.replace("XXXXX", message_id) all_log_events = self._get_log(filter_pattern, log_group_name, created_at) + current_app.logger.info(f"NUMBER OF ALL LOG EVENTS {len(all_log_events)}") if all_log_events and len(all_log_events) > 0: + current_app.logger.info( + "SHOULD RETURN SUCCESS BECAUSE WE FOUND A SUCCESS MESSAGE FOR MESSAGE ID" + ) event = all_log_events[0] message = json.loads(event["message"]) + current_app.logger.info(f"MESSAGE {message}") return "success", message["delivery"]["providerResponse"] log_group_name = ( f"sns/{region}/{account_number}/DirectPublishToPhoneNumber/Failure" ) + current_app.logger.info(f"FAILURE LOG GROUP NAME {log_group_name}") all_failed_events = self._get_log(filter_pattern, log_group_name, created_at) + current_app.logger.info( + f"NUMBER OF ALL FAILED LOG EVENTS {len(all_failed_events)}" + ) if all_failed_events and len(all_failed_events) > 0: + current_app.logger.info("SHOULD RETURN FAILED BECAUSE WE FOUND A FAILURE") event = all_failed_events[0] message = json.loads(event["message"]) + current_app.logger.info(f"MESSAGE {message}") return "failure", message["delivery"]["providerResponse"] + print(f"RAISING EXCEPTION FOR MESSAGE_ID {message_id}") raise Exception( f"No event found for message_id {message_id} notification_id {notification_id}" )