Add quotes to values in one off

Update Python Dependencies - 7/5/2024

.ds.baseline

@@ -0,0 +1,706 @@
+{
+  "version": "1.5.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "OpenAIDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".ds.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {
+    ".github/workflows/checks.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".github/workflows/checks.yml",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 61,
+        "is_secret": false
+      },
+      {
+        "type": "Basic Auth Credentials",
+        "filename": ".github/workflows/checks.yml",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 95,
+        "is_secret": false
+      }
+    ],
+    "app/assets/js/uswds.min.js": [
+      {
+        "type": "Secret Keyword",
+        "filename": "app/assets/js/uswds.min.js",
+        "hashed_secret": "372ea08cab33e71c02c651dbc83a474d32c676ea",
+        "is_verified": false,
+        "line_number": 85,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "app/assets/js/uswds.min.js",
+        "hashed_secret": "53e07a32bf191d6917ee6fd863f0b52632a86798",
+        "is_verified": false,
+        "line_number": 85,
+        "is_secret": false
+      }
+    ],
+    "app/config.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "app/config.py",
+        "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
+        "is_verified": false,
+        "line_number": 117,
+        "is_secret": false
+      }
+    ],
+    "app/main/_commonly_used_passwords.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "82e19fa12aab7cfc718a002fc82c0f074bf070e7",
+        "is_verified": false,
+        "line_number": 123,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "a172ffc990129fe6f68b50f6037c54a1894ee3fd",
+        "is_verified": false,
+        "line_number": 240,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "4de69ee6b12b7fc91070873b71ba6e2929b90619",
+        "is_verified": false,
+        "line_number": 244,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "370194ff6e0f93a7432e16cc9badd9427e8b4e13",
+        "is_verified": false,
+        "line_number": 284,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "3dd635a808ddb6dd4b6731f7c409d53dd4b14df2",
+        "is_verified": false,
+        "line_number": 356,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
+        "is_verified": false,
+        "line_number": 374,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "61d6504733ca7757e259c644acd085c4dd471019",
+        "is_verified": false,
+        "line_number": 910,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "4ea872dfd7eefbde0036da7f0780826353dc7477",
+        "is_verified": false,
+        "line_number": 940,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "b214f706bb602c1cc2adc5c6165e73622305f4bb",
+        "is_verified": false,
+        "line_number": 1010,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "5cbabd43e49a1fedbbc3b86311aa6c8fe446abf9",
+        "is_verified": false,
+        "line_number": 1195,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "18ad10fd4a67f21fc07b1aa5046b410f6b2bedf1",
+        "is_verified": false,
+        "line_number": 1213,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "10470c3b4b1fed12c3baac014be15fac67c6e815",
+        "is_verified": false,
+        "line_number": 1263,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "65e1946c8f102eca8ba0af291f7c5e807516d94c",
+        "is_verified": false,
+        "line_number": 1346,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "0075df0a74c07ee295c98238c018401c9a80183b",
+        "is_verified": false,
+        "line_number": 1397,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "ca0023d7b345802fbc227b902cb9c57a3e02195f",
+        "is_verified": false,
+        "line_number": 1442,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "c8c6ca2e11c2dfd2a40914585b5944bffea15c8c",
+        "is_verified": false,
+        "line_number": 1555,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "b85b97a99eab8c809570c61d6404c1e49bdefbb4",
+        "is_verified": false,
+        "line_number": 1596,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "dec7dd342a499dfd4d283d872ccf598d8a7b6039",
+        "is_verified": false,
+        "line_number": 1789,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "2dc5053699a351121bf839c446bd4a878dda5735",
+        "is_verified": false,
+        "line_number": 1939,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "e5d54f0ac13abbdaa94b696c2469148b96dd11ab",
+        "is_verified": false,
+        "line_number": 2242,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "6059f42e2bbae78141e8a9e6286755ee691d5ce0",
+        "is_verified": false,
+        "line_number": 2305,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "fe703d258c7ef5f50b71e06565a65aa07194907f",
+        "is_verified": false,
+        "line_number": 2348,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "c229b68e1c3ffd9874838b5cb5354a0ee1367ddc",
+        "is_verified": false,
+        "line_number": 2349,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "756de479126e911b6f3400ae686d663d9d26b509",
+        "is_verified": false,
+        "line_number": 2920,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "6b174322afcdb440ee9cc3cc11eb16f9a00dec04",
+        "is_verified": false,
+        "line_number": 2975,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "9860783bfb510cbb2bf34471ec0b84a7ea587695",
+        "is_verified": false,
+        "line_number": 3359,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "b227cbd22eaa96019ebfc4aff35ad2add2a47439",
+        "is_verified": false,
+        "line_number": 3590,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "381d48209aecab8834eb495c5b5406100da07882",
+        "is_verified": false,
+        "line_number": 3811,
+        "is_secret": false
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "app/main/_commonly_used_passwords.py",
+        "hashed_secret": "508b38590a90d32990aadd7350d160b795c3ab41",
+        "is_verified": false,
+        "line_number": 3850,
+        "is_secret": false
+      }
+    ],
+    "app/main/views/sign_in.py": [
+      {
+        "type": "Private Key",
+        "filename": "app/main/views/sign_in.py",
+        "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
+        "is_verified": false,
+        "line_number": 27,
+        "is_secret": false
+      }
+    ],
+    "app/templates/new/components/head.html": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "app/templates/new/components/head.html",
+        "hashed_secret": "ee5048791fc7ff45a1545e24f85bec3317371327",
+        "is_verified": false,
+        "line_number": 34,
+        "is_secret": false
+      }
+    ],
+    "app/templates/old/admin_template.html": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "app/templates/old/admin_template.html",
+        "hashed_secret": "ee5048791fc7ff45a1545e24f85bec3317371327",
+        "is_verified": false,
+        "line_number": 18,
+        "is_secret": false
+      }
+    ],
+    "deploy-config/sandbox.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "deploy-config/sandbox.yml",
+        "hashed_secret": "113151dd10316fcb0d5507b6215d78e2f3fe9e54",
+        "is_verified": false,
+        "line_number": 8,
+        "is_secret": false
+      }
+    ],
+    "pytest.ini": [
+      {
+        "type": "Secret Keyword",
+        "filename": "pytest.ini",
+        "hashed_secret": "577a4c667e4af8682ca431857214b3a920883efc",
+        "is_verified": false,
+        "line_number": 7,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "pytest.ini",
+        "hashed_secret": "d347784b1ab6074a65cda7bc42f1561bed85493f",
+        "is_verified": false,
+        "line_number": 7,
+        "is_secret": false
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "pytest.ini",
+        "hashed_secret": "ed1754d5cc82c8fd83205ebfb8c43fe4e88415a4",
+        "is_verified": false,
+        "line_number": 9,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "pytest.ini",
+        "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
+        "is_verified": false,
+        "line_number": 11,
+        "is_secret": false
+      }
+    ],
+    "tests/__init__.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/__init__.py",
+        "hashed_secret": "f8377c90fcfd699f0ddbdcb30c2c9183d2d933ea",
+        "is_verified": false,
+        "line_number": 388,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/forms/test_register_user_form.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/forms/test_register_user_form.py",
+        "hashed_secret": "8c6c978dc8e08771c7dea1ea2370fdf2446e5ba5",
+        "is_verified": false,
+        "line_number": 38,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/test_errorhandlers.py": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tests/app/main/test_errorhandlers.py",
+        "hashed_secret": "005fa73b3f2be8f0d71d361c1f0a9d787cd09b4e",
+        "is_verified": false,
+        "line_number": 33,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/test_request_header.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/test_request_header.py",
+        "hashed_secret": "6866ef97a972ba3a2c6ff8bb2812981054770162",
+        "is_verified": false,
+        "line_number": 21,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/organizations/test_organization_invites.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/organizations/test_organization_invites.py",
+        "hashed_secret": "bdbb156d25d02fd7792865824201dda1c60f4473",
+        "is_verified": false,
+        "line_number": 265,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/organizations/test_organization_invites.py",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 273,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_accept_invite.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_accept_invite.py",
+        "hashed_secret": "07f0a6c13923fc3b5f0c57ffa2d29b715eb80d71",
+        "is_verified": false,
+        "line_number": 607,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_new_password.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_new_password.py",
+        "hashed_secret": "a41d5c3bbcd0b39c627b9cbf4897c6d25efa694f",
+        "is_verified": false,
+        "line_number": 89,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_register.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_register.py",
+        "hashed_secret": "bdbb156d25d02fd7792865824201dda1c60f4473",
+        "is_verified": false,
+        "line_number": 116,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_register.py",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 192,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_register.py",
+        "hashed_secret": "bb5b7caa27d005d38039e3797c3ddb9bcd22c3c8",
+        "is_verified": false,
+        "line_number": 260,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_sign_in.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_sign_in.py",
+        "hashed_secret": "8b8b69116ee882b5e987e330f55db81aba0636f9",
+        "is_verified": false,
+        "line_number": 90,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_two_factor.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_two_factor.py",
+        "hashed_secret": "dc66ad927c29e31c6c374231f57a4684b0687bfe",
+        "is_verified": false,
+        "line_number": 267,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_user_profile.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_user_profile.py",
+        "hashed_secret": "8072d7aad32964ec43fbcb699c75dc38890792f7",
+        "is_verified": false,
+        "line_number": 350,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_user_profile.py",
+        "hashed_secret": "4c9dbb972da179e4f66f023eaa5fb9451d835030",
+        "is_verified": false,
+        "line_number": 351,
+        "is_secret": false
+      }
+    ],
+    "tests/app/main/views/test_verify.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/main/views/test_verify.py",
+        "hashed_secret": "faafcfa63e128929409bf310b7ea5a415f2331ce",
+        "is_verified": false,
+        "line_number": 160,
+        "is_secret": false
+      }
+    ],
+    "tests/app/notify_client/test_user_client.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/notify_client/test_user_client.py",
+        "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
+        "is_verified": false,
+        "line_number": 55,
+        "is_secret": false
+      }
+    ],
+    "tests/app/test_cloudfoundry_config.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/test_cloudfoundry_config.py",
+        "hashed_secret": "5e44dae2de8b6e57c797b968035265c9f2cd2b3e",
+        "is_verified": false,
+        "line_number": 12,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/app/test_cloudfoundry_config.py",
+        "hashed_secret": "e5e178db7317356946d13e5d2da037d39ac61c71",
+        "is_verified": false,
+        "line_number": 27,
+        "is_secret": false
+      }
+    ],
+    "tests/conftest.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/conftest.py",
+        "hashed_secret": "f8377c90fcfd699f0ddbdcb30c2c9183d2d933ea",
+        "is_verified": false,
+        "line_number": 3266,
+        "is_secret": false
+      }
+    ],
+    "tests/notifications_utils/clients/antivirus/test_antivirus_client.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/notifications_utils/clients/antivirus/test_antivirus_client.py",
+        "hashed_secret": "932b25270abe1301c22c709a19082dff07d469ff",
+        "is_verified": false,
+        "line_number": 16,
+        "is_secret": false
+      }
+    ],
+    "tests/notifications_utils/clients/encryption/test_encryption_client.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/notifications_utils/clients/encryption/test_encryption_client.py",
+        "hashed_secret": "f1e923a9667de11be6a210849a8651c1bfd81605",
+        "is_verified": false,
+        "line_number": 13,
+        "is_secret": false
+      }
+    ],
+    "tests/notifications_utils/clients/zendesk/test_zendesk_client.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/notifications_utils/clients/zendesk/test_zendesk_client.py",
+        "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
+        "is_verified": false,
+        "line_number": 16,
+        "is_secret": false
+      }
+    ]
+  },
+  "generated_at": "2024-06-11T22:26:18Z"
+}

.github/actions/setup-project/action.yml

@@ -9,10 +9,10 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.12
+    - name: Set up Python 3.12.3
       uses: actions/setup-python@v4
       with:
-        python-version: "3.12"
+        python-version: "3.12.3"
     - name: Install poetry
       shell: bash
       run: pip install poetry

.github/pull_request_template.md

@@ -1,79 +1,22 @@
 <!--
-Please follow the instructions found in this pull request template so that we
-have all of the relevant details needed for our work.
-
-At the minimum, please be sure to fill in all sections found below and also do
-the following:
-
-- Provide an appropriate and descriptive title for the pull request
-- Link the pull request to its corresponding issue (must be done after creating
-  the pull request itself)
-- Assign yourself as the author
-- Attach the appropriate labels to it
-- Set it to be on the Notify.gov project board
-- Select one or more reviewers from the team or mark the pull request as a draft
-  depending on its current state
-   - If the pull request is a draft, please be sure to add reviewers once it is
-     ready for review and mark it ready for review
-
-For each section, please delete the instructions/sample text (that includes this
-text, though it is wrapped in an HTML comment just in case) and put in your own
-information.  Thank you!
+Not sure what you should include or write in a pull request?  Please read the
+[pull request documentation in our docs!](https://github.com/GSA/notifications-api/blob/main/docs/all.md#pull-requests)
 -->
 
-*A note to PR reviewers: it may be helpful to review our
-[code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews)
-to know what to keep in mind while reviewing pull requests.*
+*A note to PR reviewers: it may be helpful to review our [code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews) to know what to keep in mind while reviewing pull requests.*
 
 ## Description
 
-Please enter a clear description about your proposed changes and what the
-expected outcome(s) is/are from there.  If there are complex implementation
-details within the changes, this is a great place to explain those details using
-plain language.
-
-This should include:
-
-- Links to issues that this PR addresses
-- Screenshots or screen captures of any visible changes, especially for UI work
-- Dependency changes
-
-If there are any caveats, known issues, follow-up items, etc., make a quick note
-of them here as well, though more details are probably warranted in the issue
-itself in this case.
+Please enter a detailed description here.
 
 ## TODO (optional)
 
-If you're opening a draft PR, it might be helpful to list any outstanding work,
-especially if you're asking folks to take a look before it's ready for full
-review.  In this case, create a small checklist with the outstanding items:
-
-- [ ] TODO item 1
-- [ ] TODO item 2
-- [ ] TODO item ...
+* [ ] TODO item 1
+* [ ] TODO item 2
+* [ ] TODO item ...
 
 ## Security Considerations
 
-Please think about the security compliance aspect of your changes and what the
-potential impacts might be.
-
-**NOTE:  Please be mindful of sharing sensitive information here!  If you're not
-sure of what to write, please ask the team first before writing anything here.**
-
-Relevant details could include (and are not limited to) the following:
-
-- Handling secrets/credential management (or specifically calling out that there
-  is nothing to handle)
-- Any adjustments to the flow of data in and out the system, or even within it
-- Connecting or disconnecting any external services to the application
-- Handling of any sensitive information, such as PII
-- Handling of information within log statements or other application monitoring
-  services/hooks
-- The inclusion of a new external dependency or the removal of an existing one
-- ... (anything else relevant from a security compliance perspective)
-
-There are some cases where there are no security considerations to be had, e.g.,
-updating our documentation with publicly available information.  In those cases
-it is fine to simply put something like this:
-
-- None; this is a documentation update with publicly available information.
+* Consideration 1
+* Consideration 2
+* Consideration ...

.github/workflows/checks.yml

@@ -157,7 +157,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
         run: poetry export --without-hashes --format=requirements.txt > requirements.txt
-      - uses: pypa/gh-action-pip-audit@v1.0.6
+      - uses: pypa/gh-action-pip-audit@v1.0.8
         with:
           inputs: requirements.txt
       - name: Run npm audit

.github/workflows/codeql.yml

@@ -0,0 +1,95 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", "production" ]
+  pull_request:
+    branches: [ "main", "production" ]
+  schedule:
+    - cron: '18 5 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: javascript-typescript
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/deploy-demo.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/demo
-            terraform/shared
+            terraform/demo/**
+            terraform/shared/**
             .github/workflows/deploy-demo.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-demo.*.acl

.github/workflows/deploy-prod.yml

@@ -18,11 +18,11 @@ jobs:
 
       - name: Check for changes to Terraform
         id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
-            terraform/production
-            terraform/shared
+            terraform/production/**
+            terraform/shared/**
             .github/workflows/deploy-prod.yml
       - name: Terraform init
         if: steps.changed-terraform-files.outputs.any_changed == 'true'
@@ -88,7 +88,7 @@ jobs:
 
       - name: Check for changes to egress config
         id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
+        uses: tj-actions/changed-files@v44
         with:
           files: |
             deploy-config/egress_proxy/notify-admin-production.*.acl

.github/workflows/deploy.yml

@@ -17,96 +17,96 @@ jobs:
 
     environment: staging
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
 
-      - name: Check for changes to Terraform
-        id: changed-terraform-files
-        uses: tj-actions/changed-files@v41.0.0
-        with:
-          files: |
-            terraform/staging
-            terraform/shared
-            .github/workflows/deploy.yml
-      - name: Terraform init
-        if: steps.changed-terraform-files.outputs.any_changed == 'true'
-        working-directory: terraform/staging
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-      - name: Terraform apply
-        if: steps.changed-terraform-files.outputs.any_changed == 'true'
-        working-directory: terraform/staging
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform apply -auto-approve -input=false
+    - name: Check for changes to Terraform
+      id: changed-terraform-files
+      uses: tj-actions/changed-files@v44
+      with:
+        files: |
+          terraform/staging/**
+          terraform/shared/**
+          .github/workflows/deploy.yml
+    - name: Terraform init
+      if: steps.changed-terraform-files.outputs.any_changed == 'true'
+      working-directory: terraform/staging
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
+      run: terraform init
+    - name: Terraform apply
+      if: steps.changed-terraform-files.outputs.any_changed == 'true'
+      working-directory: terraform/staging
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
+        TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
+        TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
+      run: terraform apply -auto-approve -input=false
 
-      - uses: ./.github/actions/setup-project
+    - uses: ./.github/actions/setup-project
 
-      - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+    - name: Create requirements.txt
+      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
 
 
-      - name: Deploy to cloud.gov
-        uses: 18f/cg-deploy-action@main
-        env:
-          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-          SECRET_KEY: ${{ secrets.SECRET_KEY }}
-          ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
-          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-          NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
-          COMMIT_HASH: ${{ github.sha }}
-          LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
-          LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
-          LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
-          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
-          LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
-          LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
-        with:
-          cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-          cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio
-          cf_space: notify-staging
-          push_arguments: >-
-            --vars-file deploy-config/staging.yml
-            --var DANGEROUS_SALT="$DANGEROUS_SALT"
-            --var SECRET_KEY="$SECRET_KEY"
-            --var ADMIN_CLIENT_USERNAME="notify-admin"
-            --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-            --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-            --var NR_BROWSER_KEY="$NR_BROWSER_KEY"
-            --var COMMIT_HASH="$COMMIT_HASH"
-            --var LOGIN_PEM="$LOGIN_PEM"
-            --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
-            --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
-            --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
-            --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
-            --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
-            --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
-            --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
+    - name: Deploy to cloud.gov
+      uses: 18f/cg-deploy-action@main
+      env:
+        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
+        SECRET_KEY: ${{ secrets.SECRET_KEY }}
+        ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+        NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
+        COMMIT_HASH: ${{ github.sha }}
+        LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
+        LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
+        LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
+        LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
+        LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
+        LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
+        LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
+        LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE"
+      with:
+        cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
+        cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
+        cf_org: gsa-tts-benefits-studio
+        cf_space: notify-staging
+        push_arguments: >-
+          --vars-file deploy-config/staging.yml
+          --var DANGEROUS_SALT="$DANGEROUS_SALT"
+          --var SECRET_KEY="$SECRET_KEY"
+          --var ADMIN_CLIENT_USERNAME="notify-admin"
+          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
+          --var NR_BROWSER_KEY="$NR_BROWSER_KEY"
+          --var COMMIT_HASH="$COMMIT_HASH"
+          --var LOGIN_PEM="$LOGIN_PEM"
+          --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID"
+          --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL"
+          --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL"
+          --var LOGIN_DOT_GOV_LOGOUT_URL="$LOGIN_DOT_GOV_LOGOUT_URL"
+          --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL"
+          --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT"
+          --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL"
 
 
-      - name: Check for changes to egress config
-        id: changed-egress-config
-        uses: tj-actions/changed-files@v41.0.0
-        with:
-          files: |
-            deploy-config/egress_proxy/notify-admin-staging.*.acl
-            .github/actions/deploy-proxy/action.yml
-            .github/workflows/deploy.yml
-      - name: Deploy egress proxy
-        if: steps.changed-egress-config.outputs.any_changed == 'true'
-        uses: ./.github/actions/deploy-proxy
-        with:
-          cf_space: notify-staging
-          app: notify-admin-staging
+    - name: Check for changes to egress config
+      id: changed-egress-config
+      uses: tj-actions/changed-files@v44
+      with:
+        files: |
+          deploy-config/egress_proxy/notify-admin-staging.*.acl
+          .github/actions/deploy-proxy/action.yml
+          .github/workflows/deploy.yml
+    - name: Deploy egress proxy
+      if: steps.changed-egress-config.outputs.any_changed == 'true'
+      uses: ./.github/actions/deploy-proxy
+      with:
+        cf_space: notify-staging
+        app: notify-admin-staging
 
   bail:
     runs-on: ubuntu-latest

.nvmrc

@@ -1 +1 @@
-16.15.1
+22.3.0

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v4.6.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -11,3 +11,14 @@ repos:
     -   id: debug-statements
     -   id: check-merge-conflict
     -   id: check-toml
+    -   id: check-ast
+    -   id: fix-byte-order-marker
+    -   id: detect-aws-credentials
+        args: [--allow-missing-credentials]
+    -   id: detect-private-key
+    -   id: mixed-line-ending
+-   repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
+    hooks:
+    -   id: detect-secrets
+        args: ['--baseline', '.ds.baseline']

Makefile

@@ -21,7 +21,8 @@ bootstrap: generate-version-file ## Set up everything to run the app
 	poetry install --sync --no-root
 	poetry run playwright install --with-deps
 	poetry run pre-commit install
-	source $(NVMSH) --no-use && nvm install && npm ci --no-audit
+	source $(NVMSH) --no-use && nvm install && npm install
+	source $(NVMSH) && npm ci --no-audit
 	source $(NVMSH) && npm run build
 
 .PHONY: watch-frontend
@@ -40,6 +41,10 @@ run-flask-bare:  ## Run flask without invoking poetry so we can override ENV var
 npm-audit:  ## Check for vulnerabilities in NPM packages
 	source $(NVMSH) && npm run audit
 
+.PHONY: npm-audit-fix
+npm-audit-fix:  ## Fix vulnerabilities that do not require attentino (according to npm)
+	source $(NVMSH) && npm audit fix
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -71,18 +76,19 @@ too-complex:
 .PHONY: py-test
 py-test: export NEW_RELIC_ENVIRONMENT=test
 py-test: ## Run python unit tests
-	poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10 --ignore=tests/end_to_end tests/
+	poetry run coverage run -m pytest --maxfail=10 --ignore=tests/end_to_end tests/
 	poetry run coverage report --fail-under=96
 	poetry run coverage html -d .coverage_cache
 
 .PHONY: dead-code
-dead-code:
-	poetry run vulture ./app --min-confidence=100
+dead-code: ## 60% is our aspirational goal, but currently breaks the build
+	poetry run vulture ./app ./notifications_utils --min-confidence=100
+
 
 .PHONY: e2e-test
 e2e-test: export NEW_RELIC_ENVIRONMENT=test
 e2e-test: ## Run end-to-end integration tests; note that --browser webkit isn't currently working
-	poetry run pytest -v --browser chromium --browser firefox tests/end_to_end
+	poetry run pytest -vv --browser chromium  --browser firefox tests/end_to_end
 
 .PHONY: js-lint
 js-lint: ## Run javascript linting scanners
@@ -101,13 +107,6 @@ py-lock: ## Syncs dependencies and updates lock file without performing recursiv
 	poetry lock --no-update
 	poetry install --sync
 
-.PHONY: update-utils
-update-utils: ## Forces Poetry to pull the latest changes from the notifications-utils repo; requires that you commit the changes to poetry.lock!
-	poetry update notifications-utils
-	@echo
-	@echo !!! PLEASE MAKE SURE TO COMMIT AND PUSH THE UPDATED poetry.lock FILE !!!
-	@echo
-
 .PHONY: freeze-requirements
 freeze-requirements: ## create static requirements.txt
 	poetry export --without-hashes --format=requirements.txt > requirements.txt

README.md

@@ -19,7 +19,6 @@ UI's backend and is required for most things to function. Set that up first!
 Our other repositories are:
 
 - [notifications-admin](https://github.com/GSA/notifications-admin)
-- [notifications-utils](https://github.com/GSA/notifications-utils)
 - [us-notify-compliance](https://github.com/GSA/us-notify-compliance/)
 - [notify-python-demo](https://github.com/GSA/notify-python-demo)
 
@@ -41,7 +40,7 @@ You will need the following items:
 [Follow the instructions here to set up the Notify.gov API.](https://github.com/GSA/notifications-api#before-you-start)
 
 The Notify.gov API is required in order for the Notify.gov Admin UI to run, and
-it will also take care of many of the steps that are listed here.  The sections
+it will also take care of many of the steps that are listed here. The sections
 that are a repeat from the API setup are flagged with an **[API Step]** label
 in front of them.
 
@@ -49,7 +48,7 @@ in front of them.
 
 This project is set up to work with
 [nvm (Node Version Manager)](https://github.com/nvm-sh/nvm#installing-and-updating)
-for managing and using Node.js (version 16.15.1) and the `npm` package manager.
+for managing and using Node.js (version 22.3.0) and the `npm` package manager.
 
 These instructions will walk you through how to set your machine up with all of
 the required tools for this project.
@@ -84,11 +83,13 @@ Your system `$PATH` environment variable is likely set in one of these
 locations:
 
 For BASH shells:
+
 - `~/.bashrc`
 - `~/.bash_profile`
 - `~/.profile`
 
 For ZSH shells:
+
 - `~/.zshrc`
 - `~/.zprofile`
 
@@ -98,7 +99,7 @@ environments.
 Which file you need to modify depends on whether or not you are running an
 interactive shell or a login shell
 (see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell)
-for an explanation of the differences).  If you're still not sure, please ask
+for an explanation of the differences). If you're still not sure, please ask
 the team for help!
 
 Once you determine which file you'll need to modify, add these lines before any
@@ -159,7 +160,7 @@ _NOTE: This project currently uses the latest `1.4.x release of Terraform._
 #### [API Step] Python Installation
 
 Now we're going to install a tool to help us manage Python versions and
-virtual environments on our system.  First, we'll install
+virtual environments on our system. First, we'll install
 [pyenv](https://github.com/pyenv/pyenv) and one of its plugins,
 [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew:
 
@@ -286,7 +287,7 @@ we'll use `3.12` in our example here since we recently upgraded to this version:
 pyenv install 3.12
 ```
 
-Next, delete the virtual environment you previously had set up.  If you followed
+Next, delete the virtual environment you previously had set up. If you followed
 the instructions above with the first-time set up, you can do this with `pyenv`:
 
 ```sh
@@ -307,6 +308,20 @@ you'll be set with an upgraded version of Python.
 
 _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
 
+#### Updating the .env file for Login.gov
+
+To configure the application for Login.gov, you will need to update the following environment variables in the .env file:
+
+```
+COMMIT_HASH=”--------”
+```
+
+Reach out to someone on the team to get the most recent Login.gov key.
+
+```
+LOGIN_PEM="INSERT_LOGIN_GOV_KEY_HERE"
+```
+
 #### Updating the .env file for E2E tests
 
 With the newly created `.env` file in place, you'll need to make one more
@@ -351,6 +366,28 @@ This will run the local development web server and make the admin site
 available at http://localhost:6012; remember to make sure that the Notify.gov
 API is running as well!
 
+## Creating a 'First User' in the database
+
+After you have completed all setup steps, you will be unable to log in, because there
+will not be a user in the database to link to the login.gov account you are using. So
+you will need to create that user in your database using the 'create-test-user' command.
+
+Open two terminals pointing to the api project and then run these commands in the
+respective terminals.
+
+(Server 1)
+env ALLOW_EXPIRED_API_TOKEN=1 make run-flask
+
+(Server 2)
+poetry run flask command create-admin-jwt | tail -n 1 | pbcopy
+poetry run flask command create-test-user --admin=True;
+
+Supply your name, email address, mobile number, and password when prompted. Make sure the email address
+is the same one you are using in login.gov and make sure your phone number is in the format 5555555555.
+
+If for any reason in the course of development it is necessary for your to delete your db
+via the `dropdb` command, you will need to repeat these steps when you recreate your db.
+
 ## Git Hooks
 
 We're using [`pre-commit`](https://pre-commit.com/) to manage hooks in order to
@@ -396,23 +433,6 @@ In either situation, once you are finished and have verified the dependency
 changes are working, please be sure to commit both the `pyproject.toml` and
 `poetry.lock` files.
 
-### Keeping the notification-utils Dependency Up-to-Date
-
-The `notifications-utils` dependency references the other repository we have at
-https://github.com/GSA/notifications-utils - this dependency requires a bit of
-extra legwork to ensure it stays up-to-date.
-
-Whenever a PR is merged in the `notifications-utils` repository, we need to make
-sure the changes are pulled in here and committed to this repository as well.
-You can do this by going through these steps:
-
-- Make sure your local `main` branch is up-to-date
-- Create a new branch to work in
-- Run `make update-utils`
-- Commit the updated `poetry.lock` file and push the changes
-- Make a new PR with the change
-- Have the PR get reviewed and merged
-
 ## Known Installation Issues
 
 ### Python Installation Errors

app/__init__.py

@@ -18,6 +18,7 @@ from flask import (
 )
 from flask.globals import request_ctx
 from flask_login import LoginManager, current_user
+from flask_socketio import SocketIO
 from flask_talisman import Talisman
 from flask_wtf import CSRFProtect
 from flask_wtf.csrf import CSRFError
@@ -30,7 +31,7 @@ from werkzeug.local import LocalProxy
 from app import proxy_fix
 from app.asset_fingerprinter import asset_fingerprinter
 from app.config import configs
-from app.extensions import redis_client, zendesk_client
+from app.extensions import redis_client
 from app.formatters import (
     convert_markdown_template,
     convert_to_boolean,
@@ -46,6 +47,7 @@ from app.formatters import (
     format_datetime_human,
     format_datetime_normal,
     format_datetime_relative,
+    format_datetime_scheduled_notification,
     format_datetime_table,
     format_day_of_week,
     format_delta,
@@ -117,6 +119,7 @@ from notifications_utils.recipients import format_phone_number_human_readable
 login_manager = LoginManager()
 csrf = CSRFProtect()
 talisman = Talisman()
+socketio = SocketIO()
 
 
 # The current service attached to the request stack.
@@ -174,6 +177,7 @@ def create_app(application):
 
     init_govuk_frontend(application)
     init_jinja(application)
+    socketio.init_app(application)
 
     for client in (
         csrf,
@@ -201,7 +205,6 @@ def create_app(application):
         user_api_client,
         # External API clients
         redis_client,
-        zendesk_client,
     ):
         client.init_app(application)
 
@@ -230,6 +233,24 @@ def create_app(application):
     )
     logging.init_app(application)
 
+    # Hopefully will help identify if there is a race condition causing the CSRF errors
+    # that we have occasionally seen in our environments.
+    for key in ("SECRET_KEY", "DANGEROUS_SALT"):
+        try:
+            value = application.config[key]
+        except KeyError:
+            application.logger.error(f"Env Var {key} doesn't exist.")
+        else:
+            try:
+                data_len = len(value.strip())
+            except (TypeError, AttributeError):
+                application.logger.error(f"Env Var {key} invalid type: {type(value)}")
+            else:
+                if data_len:
+                    application.logger.info(f"Env Var {key} is a non-zero length.")
+                else:
+                    application.logger.error(f"Env Var {key} is empty.")
+
     login_manager.login_view = "main.sign_in"
     login_manager.login_message_category = "default"
     login_manager.session_protection = None
@@ -551,6 +572,7 @@ def add_template_filters(application):
         format_datetime,
         format_datetime_24h,
         format_datetime_normal,
+        format_datetime_scheduled_notification,
         format_datetime_table,
         valid_phone_number,
         linkable_name,

app/assets/images/logo-login.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 292.1 39.47"><path fill="#fff" d="M7.44 6.89h17.2v22.06H7.44z"/><path fill="#112e51" d="M53.45 5.28h-6.16v27.31h17.87v-5.38H53.45V5.28zm30.76-.19c-8.35 0-14.67 5.89-14.67 13.85s6.32 13.93 14.67 13.93 14.67-6 14.67-13.93S92.56 5.09 84.21 5.09zm.08 22.39a8.49 8.49 0 0 1-8.43-8.54 8.4 8.4 0 0 1 8.43-8.47 8.29 8.29 0 0 1 8.27 8.47 8.36 8.36 0 0 1-8.27 8.54zm35.36-17.05a11.43 11.43 0 0 1 7.49 3l3.47-4.41a17.21 17.21 0 0 0-11-4c-8.51 0-14.9 6-14.9 13.85s6.29 13.99 14.58 13.99a20.21 20.21 0 0 0 11.08-3.55V17.57h-10.11v4.82H125V26a12.42 12.42 0 0 1-5.34 1.48 8.52 8.52 0 1 1 0-17zm19.61-5.15h6.17v27.31h-6.17zm34.53 17.56L160.52 5.28h-5.61v27.31h5.89V15.07l13.22 17.52h5.62V5.28h-5.85v17.56zm17.47 2.85a3.39 3.39 0 0 0-3.47 3.56 3.47 3.47 0 1 0 6.94 0 3.36 3.36 0 0 0-3.47-3.56zm23.15-15.26a11.43 11.43 0 0 1 7.49 3l3.47-4.41a17.21 17.21 0 0 0-11-4c-8.51 0-14.9 6-14.9 13.85s6.28 13.93 14.55 13.93a20.21 20.21 0 0 0 11.08-3.55V17.57H215v4.82h4.77V26a12.42 12.42 0 0 1-5.34 1.48 8.52 8.52 0 1 1 0-17zm32.23-5.34C238.29 5.09 232 11 232 18.94s6.32 13.93 14.67 13.93 14.67-6 14.67-13.93S255 5.09 246.64 5.09zm.08 22.39a8.49 8.49 0 0 1-8.43-8.54 8.4 8.4 0 0 1 8.43-8.47 8.29 8.29 0 0 1 8.28 8.47 8.36 8.36 0 0 1-8.28 8.54zm39.02-22.2-7.49 20.6-7.45-20.6h-6.63l10.73 27.31h6.36L292.1 5.28h-6.36z"/><path fill="#e21d3e" d="m11.38 25.34 1.45-8.28a4.39 4.39 0 0 1 2.71-7.86V0H4.34A4.34 4.34 0 0 0 0 4.34v21.28a2.62 2.62 0 0 0 .26 1.16c.94 1.9 4.65 8 15.27 12.69V25.7a24.3 24.3 0 0 1-4.15-.36z"/><path fill="#b51e23" d="M26.72 0H15.53v9.19A4.39 4.39 0 0 1 18.28 17l1.45 8.32a24.29 24.29 0 0 1-4.2.36v13.79C26.14 34.8 29.86 28.68 30.8 26.79a2.62 2.62 0 0 0 .26-1.16V4.34A4.34 4.34 0 0 0 26.72 0z"/></svg>

app/assets/javascripts/sampleChartDashboard.js

@@ -0,0 +1,67 @@
+(function (window) {
+
+    function initializeChartAndSocket() {
+        var ctx = document.getElementById('myChart');
+        if (!ctx) {
+            return;
+        }
+
+        var myBarChart = new Chart(ctx.getContext('2d'), {
+            type: 'bar',
+            data: {
+                labels: [],
+                datasets: [
+                    {
+                        label: 'Delivered',
+                        data: [],
+                        backgroundColor: '#0076d6',
+                        stack: 'Stack 0'
+                    },
+                ]
+            },
+            options: {
+                scales: {
+                    y: {
+                        beginAtZero: true
+                    }
+                }
+            }
+        });
+
+        var socket = io();
+
+        socket.on('connect', function() {
+            socket.emit('fetch_daily_stats_by_user');
+        });
+
+        socket.on('daily_stats_by_user_update', function(data) {
+            // console.log('Data received:', data);
+            var labels = [];
+            var deliveredData = [];
+
+            for (var date in data) {
+                labels.push(date);
+                deliveredData.push(data[date].sms.delivered);
+            }
+
+            myBarChart.data.labels = labels;
+            myBarChart.data.datasets[0].data = deliveredData;
+            myBarChart.update();
+        });
+
+        socket.on('error', function(data) {
+            // console.log('Error:', data);
+        });
+
+        var sevenDaysButton = document.getElementById('sevenDaysButton');
+        if (sevenDaysButton) {
+            sevenDaysButton.addEventListener('click', function() {
+                socket.emit('fetch_daily_stats_by_user');
+                // console.log('clicked');
+            });
+        }
+    }
+
+    document.addEventListener('DOMContentLoaded', initializeChartAndSocket);
+
+})(window);

app/assets/sass/uswds/_legacy-styles.scss

@@ -66,6 +66,10 @@ h2.sms-message-header {
   margin-bottom: 0.5rem;
 }
 
+.usa-prose >*+ h2.message-header {
+  margin-top: 1em;
+}
+
 h2.recipient-list {
   margin-bottom: 0.5rem;
 }

app/assets/sass/uswds/_uswds-theme-custom-styles.scss

@@ -157,6 +157,18 @@ td.table-empty-message {
    }
 }
 
+.usa-button img {
+   margin-left: .5rem;
+   height: 1rem;
+}
+
+.usa-button.login-button.login-button--primary,.login-button.login-button--primary:hover{
+   color:#112e51;background-color:#fff;
+   border:1px solid #767676;
+   display: inline-flex;
+   justify-content: center;
+}
+
 .user-list-edit-link:active:before,
 .user-list-edit-link:focus:before {
    box-shadow: none;
@@ -583,3 +595,9 @@ details form {
 #countdown-container {
    display: none; // Hide the countdown timer
 }
+
+.placeholder, .placeholder-conditional {
+   background-color: #face00;
+   border: 1px solid #face00;
+   border-radius: 7px;
+}

app/config.py

@@ -38,7 +38,7 @@ class Config(object):
     NR_MONITOR_ON = settings and settings.monitor_mode
     COMMIT_HASH = getenv("COMMIT_HASH", "--------")[0:7]
 
-    GOVERNMENT_EMAIL_DOMAIN_NAMES = ["gov"]
+    GOVERNMENT_EMAIL_DOMAIN_NAMES = ["gov", "mil", "si.edu"]
 
     # Logging
     NOTIFY_LOG_LEVEL = getenv("NOTIFY_LOG_LEVEL", "INFO")
@@ -53,7 +53,13 @@ class Config(object):
     PERMANENT_SESSION_LIFETIME = 1800  # 30 Minutes
     SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60  # 1 year
     REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45
-    ACTIVITY_STATS_LIMIT_DAYS = 7
+    ACTIVITY_STATS_LIMIT_DAYS = {
+        "today": 0,
+        "one_day": 1,
+        "three_day": 3,
+        "five_day": 5,
+        "seven_day": 7,
+    }
     SESSION_COOKIE_HTTPONLY = True
     SESSION_COOKIE_NAME = "notify_admin_session"
     SESSION_COOKIE_SECURE = True

app/extensions.py

@@ -1,5 +1,3 @@
 from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 
-zendesk_client = ZendeskClient()
 redis_client = RedisClient()

app/formatters.py

@@ -13,11 +13,12 @@ import humanize
 import markdown
 import pytz
 from bs4 import BeautifulSoup
-from flask import Markup, render_template_string, url_for
+from flask import render_template_string, url_for
 from flask.helpers import get_root_path
+from markupsafe import Markup
 
 from app.utils.csv import get_user_preferred_timezone
-from app.utils.time import parse_dt, parse_naive_dt
+from app.utils.time import parse_naive_dt
 from notifications_utils.field import Field
 from notifications_utils.formatters import make_quotes_smart
 from notifications_utils.formatters import nl2br as utils_nl2br
@@ -93,16 +94,34 @@ def format_datetime_normal(date):
     )
 
 
+def format_datetime_scheduled_notification(date):
+    # e.g. April 09, 2024 at 04:00 PM US/Eastern.
+    # Everything except scheduled notifications, the time is always "now".
+    # Scheduled notifications are the exception to the rule.
+    # Here we are formating and displaying the datetime without converting datetime to a different timezone.
+
+    datetime_obj = parse_naive_dt(date)
+
+    format_time_without_tz = datetime_obj.replace(tzinfo=timezone.utc).strftime(
+        "%I:%M %p"
+    )
+    return "{} at {} {}".format(
+        format_date_normal(date), format_time_without_tz, get_user_preferred_timezone()
+    )
+
+
 def format_datetime_table(date):
     # example:  03-18-2024 at 04:53 PM, intended for datetimes in tables
     return "{} at {}".format(format_date_numeric(date), format_time_12h(date))
 
 
 def format_time_12h(date):
-    date = parse_dt(date)
+    date = parse_naive_dt(date)
 
     preferred_tz = pytz.timezone(get_user_preferred_timezone())
-    return date.astimezone(preferred_tz).strftime("%I:%M %p")
+    return (
+        date.replace(tzinfo=timezone.utc).astimezone(preferred_tz).strftime("%I:%M %p")
+    )
 
 
 def format_datetime_relative(date):

app/main/forms.py

@@ -4,12 +4,13 @@ from itertools import chain
 from numbers import Number
 
 import pytz
-from flask import Markup, render_template, request
+from flask import render_template, request
 from flask_login import current_user
 from flask_wtf import FlaskForm as Form
 from flask_wtf.file import FileAllowed
 from flask_wtf.file import FileField as FileField_wtf
 from flask_wtf.file import FileSize
+from markupsafe import Markup
 from werkzeug.utils import cached_property
 from wtforms import (
     BooleanField,

app/main/views/api_keys.py

@@ -1,5 +1,6 @@
-from flask import Markup, abort, flash, redirect, render_template, request, url_for
+from flask import abort, flash, redirect, render_template, request, url_for
 from flask_login import current_user
+from markupsafe import Markup
 
 from app import (
     api_key_api_client,

app/main/views/dashboard.py

@@ -6,6 +6,7 @@ from itertools import groupby
 
 from flask import Response, abort, jsonify, render_template, request, session, url_for
 from flask_login import current_user
+from flask_socketio import emit
 from werkzeug.utils import redirect
 
 from app import (
@@ -14,6 +15,7 @@ from app import (
     job_api_client,
     notification_api_client,
     service_api_client,
+    socketio,
     template_statistics_client,
 )
 from app.formatters import format_date_numeric, format_datetime_numeric, get_time_left
@@ -32,6 +34,38 @@ from app.utils.user import user_has_permissions
 from notifications_utils.recipients import format_phone_number_human_readable
 
 
+@socketio.on("fetch_daily_stats")
+def handle_fetch_daily_stats():
+    service_id = session.get('service_id')
+    if service_id:
+        date_range = get_stats_date_range()
+        daily_stats = service_api_client.get_service_notification_statistics_by_day(
+            service_id, start_date=date_range["start_date"], days=date_range["days"]
+        )
+        emit("daily_stats_update", daily_stats)
+    else:
+        emit("error", {"error": "No service_id provided"})
+
+
+@socketio.on("fetch_daily_stats_by_user")
+def handle_fetch_daily_stats_by_user():
+    service_id = session.get("service_id")
+    user_id = session.get("user_id")
+    if service_id and user_id:
+        date_range = get_stats_date_range()
+        daily_stats_by_user = (
+            service_api_client.get_user_service_notification_statistics_by_day(
+                service_id,
+                user_id,
+                start_date=date_range["start_date"],
+                days=date_range["days"],
+            )
+        )
+        emit("daily_stats_by_user_update", daily_stats_by_user)
+    else:
+        emit("error", {"error": "No service_id or user_id provided"})
+
+
 @main.route("/services/<uuid:service_id>/dashboard")
 @user_has_permissions("view_activity", "send_messages")
 def old_service_dashboard(service_id):
@@ -134,11 +168,17 @@ def template_usage(service_id):
         months=months,
         stats=stats,
         most_used_template_count=max(
-            max(
-                (template["requested_count"] for template in month["templates_used"]),
-                default=0,
-            )
-            for month in months
+            (
+                max(
+                    (
+                        template["requested_count"]
+                        for template in month["templates_used"]
+                    ),
+                    default=0,
+                )
+                for month in months
+            ),
+            default=0,
         ),
         years=get_tuples_of_financial_years(
             partial(url_for, ".template_usage", service_id=service_id),
@@ -155,31 +195,16 @@ def usage(service_id):
     year, current_financial_year = requested_and_current_financial_year(request)
 
     free_sms_allowance = billing_api_client.get_free_sms_fragment_limit_for_year(
-        service_id, year
+        service_id
     )
+
     units = billing_api_client.get_monthly_usage_for_service(service_id, year)
+
     yearly_usage = billing_api_client.get_annual_usage_for_service(service_id, year)
 
     more_stats = format_monthly_stats_to_list(
         service_api_client.get_monthly_notification_stats(service_id, year)["data"]
     )
-    if year == current_financial_year:
-        # This includes Oct, Nov, Dec
-        # but we don't need next year's data yet
-        more_stats = [
-            month
-            for month in more_stats
-            if month["name"] in ["October", "November", "December"]
-        ]
-    elif year == (current_financial_year + 1):
-        # This is all the other months
-        # and we need last year's data
-        more_stats = [
-            month
-            for month in more_stats
-            if month["name"] not in ["October", "November", "December"]
-        ]
-
     return render_template(
         "views/usage.html",
         months=list(get_monthly_usage_breakdown(year, units, more_stats)),
@@ -341,8 +366,15 @@ def get_dashboard_partials(service_id):
     dashboard_totals = (get_dashboard_totals(stats),)
     free_sms_allowance = billing_api_client.get_free_sms_fragment_limit_for_year(
         current_service.id,
-        get_current_financial_year(),
     )
+    # These 2 calls will update the dashboard sms allowance count while in trial mode.
+    billing_api_client.get_monthly_usage_for_service(
+        service_id, get_current_financial_year()
+    )
+    billing_api_client.create_or_update_free_sms_fragment_limit(
+        service_id, free_sms_fragment_limit=free_sms_allowance
+    )
+
     yearly_usage = billing_api_client.get_annual_usage_for_service(
         service_id,
         get_current_financial_year(),
@@ -433,13 +465,25 @@ def aggregate_status_types(counts_dict):
 
 
 def get_months_for_financial_year(year, time_format="%B"):
-    return [
-        month.strftime(time_format)
-        for month in (
-            get_months_for_year(10, 13, year) + get_months_for_year(1, 10, year + 1)
-        )
-        if month < datetime.now()
-    ]
+    return [month.strftime(time_format) for month in (get_months_for_year(1, 13, year))]
+
+
+def get_current_month_for_financial_year(year):
+    current_month = datetime.now().month
+    return current_month
+
+
+def get_stats_date_range():
+    current_financial_year = get_current_financial_year()
+    current_month = get_current_month_for_financial_year(current_financial_year)
+    start_date = datetime.now().strftime("%Y-%m-%d")
+    days = 7
+    return {
+        "current_financial_year": current_financial_year,
+        "current_month": current_month,
+        "start_date": start_date,
+        "days": days,
+    }
 
 
 def get_months_for_year(start, end, year):

app/main/views/index.py

@@ -1,6 +1,6 @@
 import os
 
-from flask import abort, redirect, render_template, request, url_for
+from flask import abort, current_app, redirect, render_template, request, url_for
 from flask_login import current_user
 
 from app import status_api_client
@@ -9,20 +9,28 @@ from app.main import main
 from app.main.views.pricing import CURRENT_SMS_RATE
 from app.main.views.sub_navigation_dictionaries import features_nav, using_notify_nav
 from app.utils.user import user_is_logged_in
-
-login_dot_gov_url = os.getenv("LOGIN_DOT_GOV_INITIAL_SIGNIN_URL")
+from notifications_utils.url_safe_token import generate_token
 
 
 @main.route("/")
 def index():
     if current_user and current_user.is_authenticated:
         return redirect(url_for("main.choose_account"))
-
+    token = generate_token(
+        str(request.remote_addr),
+        current_app.config["SECRET_KEY"],
+        current_app.config["DANGEROUS_SALT"],
+    )
+    url = os.getenv("LOGIN_DOT_GOV_INITIAL_SIGNIN_URL")
+    # handle unit tests
+    if url is not None:
+        url = url.replace("NONCE", token)
+        url = url.replace("STATE", token)
     return render_template(
         "views/signedout.html",
         sms_rate=CURRENT_SMS_RATE,
         counts=status_api_client.get_count_of_live_services_and_organizations(),
-        login_dot_gov_url=login_dot_gov_url,
+        initial_signin_url=url,
     )
 
 

app/main/views/jobs.py

@@ -3,7 +3,6 @@
 from functools import partial
 
 from flask import (
-    Markup,
     Response,
     abort,
     jsonify,
@@ -15,6 +14,7 @@ from flask import (
     url_for,
 )
 from flask_login import current_user
+from markupsafe import Markup
 
 from app import (
     current_service,
@@ -143,11 +143,40 @@ def view_notifications(service_id, message_type=None):
             True: ["reference"],
             False: [],
         }.get(bool(current_service.api_keys)),
-        download_link=url_for(
+        download_link_one_day=url_for(
             ".download_notifications_csv",
             service_id=current_service.id,
             message_type=message_type,
             status=request.args.get("status"),
+            number_of_days="one_day",
+        ),
+        download_link_today=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="today",
+        ),
+        download_link_three_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="three_day",
+        ),
+        download_link_five_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="five_day",
+        ),
+        download_link_seven_day=url_for(
+            ".download_notifications_csv",
+            service_id=current_service.id,
+            message_type=message_type,
+            status=request.args.get("status"),
+            number_of_days="seven_day",
         ),
     )
 
@@ -183,10 +212,9 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     filter_args["status"] = set_status_filters(filter_args)
     service_data_retention_days = None
     search_term = request.form.get("to", "")
-
     if message_type is not None:
         service_data_retention_days = current_service.get_days_of_retention(
-            message_type
+            message_type, number_of_days="seven_day"
         )
 
     if request.path.endswith("csv") and current_user.has_permissions("view_activity"):
@@ -212,7 +240,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
     )
     url_args = {"message_type": message_type, "status": request.args.get("status")}
     prev_page = None
-
     if "links" in notifications and notifications["links"].get("prev", None):
         prev_page = generate_previous_dict(
             "main.view_notifications", service_id, page, url_args=url_args
@@ -233,7 +260,6 @@ def get_notifications(service_id, message_type, status_override=None):  # noqa
         )
     else:
         download_link = None
-
     return {
         "service_data_retention_days": service_data_retention_days,
         "counts": render_template(
@@ -362,6 +388,7 @@ def get_job_partials(job):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
     notifications = job.get_notifications(status=filter_args["status"])
+    number_of_days = "seven_day"
     counts = render_template(
         "partials/count.html",
         counts=_get_job_counts(job),
@@ -371,7 +398,7 @@ def get_job_partials(job):
         ),
     )
     service_data_retention_days = current_service.get_days_of_retention(
-        job.template_type
+        job.template_type, number_of_days
     )
 
     if request.referrer is not None:

app/main/views/notifications.py

@@ -137,9 +137,9 @@ def get_all_personalisation_from_notification(notification):
 def download_notifications_csv(service_id):
     filter_args = parse_filter_args(request.args)
     filter_args["status"] = set_status_filters(filter_args)
-
+    number_of_days = request.args["number_of_days"]
     service_data_retention_days = current_service.get_days_of_retention(
-        filter_args.get("message_type")[0]
+        filter_args.get("message_type")[0], number_of_days
     )
     file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
     file_time = f"{file_time} {get_user_preferred_timezone()}"

app/main/views/platform_admin.py

@@ -662,7 +662,21 @@ def create_global_stats(services):
         "email": {"delivered": 0, "failed": 0, "requested": 0},
         "sms": {"delivered": 0, "failed": 0, "requested": 0},
     }
+    # Issue #1323. The back end is now sending 'failure' instead of
+    # 'failed'.  Adjust it here, but keep it flexible in case
+    # the backend reverts to 'failed'.
     for service in services:
+        if service["statistics"]["sms"].get("failure") is not None:
+            service["statistics"]["sms"]["failed"] = service["statistics"]["sms"][
+                "failure"
+            ]
+        if service["statistics"]["email"].get("failure") is not None:
+            service["statistics"]["email"]["failed"] = service["statistics"]["email"][
+                "failure"
+            ]
+
+    for service in services:
+
         for msg_type, status in itertools.product(
             ("sms", "email"), ("delivered", "failed", "requested")
         ):

app/main/views/register.py

@@ -1,9 +1,12 @@
+import base64
+import json
 import uuid
 from datetime import datetime, timedelta
 
 from flask import (
     abort,
     current_app,
+    flash,
     redirect,
     render_template,
     request,
@@ -12,18 +15,18 @@ from flask import (
 )
 from flask_login import current_user
 
-from app import user_api_client
+from app import redis_client, user_api_client
 from app.main import main
 from app.main.forms import (
     RegisterUserForm,
-    RegisterUserFromInviteForm,
     RegisterUserFromOrgInviteForm,
     SetupUserProfileForm,
 )
 from app.main.views import sign_in
 from app.main.views.verify import activate_user
 from app.models.user import InvitedOrgUser, InvitedUser, User
-from app.utils import hide_from_search_engines
+from app.utils import hide_from_search_engines, hilite
+from app.utils.user import is_gov_user
 
 
 @main.route("/register", methods=["GET", "POST"])
@@ -40,35 +43,10 @@ def register():
     return render_template("views/register.html", form=form)
 
 
-@main.route("/register-from-invite", methods=["GET", "POST"])
-def register_from_invite():
-    invited_user = InvitedUser.from_session()
-    if not invited_user:
-        abort(404)
-
-    form = RegisterUserFromInviteForm(invited_user)
-
-    if form.validate_on_submit():
-        if (
-            form.service.data != invited_user.service
-            or form.email_address.data != invited_user.email_address
-        ):
-            abort(400)
-        _do_registration(form, send_email=False, send_sms=invited_user.sms_auth)
-        invited_user.accept_invite()
-        if invited_user.sms_auth:
-            return redirect(url_for("main.verify"))
-        else:
-            # we've already proven this user has email because they clicked the invite link,
-            # so just activate them straight away
-            return activate_user(session["user_details"]["id"])
-
-    return render_template(
-        "views/register-from-invite.html", invited_user=invited_user, form=form
-    )
-
-
 @main.route("/register-from-org-invite", methods=["GET", "POST"])
+# TODO This is deprecated, we are now handling invites in the
+# login.gov workflow.  Leaving it here until we write the new
+# org registration.
 def register_from_org_invite():
     invited_org_user = InvitedOrgUser.from_session()
     if not invited_org_user:
@@ -136,31 +114,94 @@ def registration_continue():
         raise Exception("Unexpected routing in registration_continue")
 
 
+def get_invite_data_from_redis(state):
+
+    invite_data = json.loads(redis_client.get(f"invitedata-{state}"))
+    user_email = redis_client.get(f"user_email-{state}").decode("utf8")
+    user_uuid = redis_client.get(f"user_uuid-{state}").decode("utf8")
+    invited_user_email_address = redis_client.get(
+        f"invited_user_email_address-{state}"
+    ).decode("utf8")
+    return invite_data, user_email, user_uuid, invited_user_email_address
+
+
+def put_invite_data_in_redis(
+    state, invite_data, user_email, user_uuid, invited_user_email_address
+):
+    ttl = 60 * 15  # 15 minutes
+
+    redis_client.set(f"invitedata-{state}", json.dumps(invite_data), ex=ttl)
+    redis_client.set(f"user_email-{state}", user_email, ex=ttl)
+    redis_client.set(f"user_uuid-{state}", user_uuid, ex=ttl)
+    redis_client.set(
+        f"invited_user_email_address-{state}",
+        invited_user_email_address,
+        ex=ttl,
+    )
+
+
+def check_invited_user_email_address_matches_expected(
+    user_email, invited_user_email_address
+):
+    if user_email.lower() != invited_user_email_address.lower():
+        debug_msg("invited user email did not match expected email, abort(403)")
+        flash("You cannot accept an invite for another person.")
+        abort(403)
+
+    if not is_gov_user(user_email):
+        debug_msg("invited user has a non-government email address.")
+        flash("You must use a government email address.")
+        abort(403)
+
+
 @main.route("/set-up-your-profile", methods=["GET", "POST"])
 @hide_from_search_engines
 def set_up_your_profile():
 
+    debug_msg(f"Enter set_up_your_profile with request.args {request.args}")
+    code = request.args.get("code")
+    state = request.args.get("state")
+    login_gov_error = request.args.get("error")
+
+    if redis_client.get(f"invitedata-{state}") is None:
+        access_token = sign_in._get_access_token(code, state)
+        debug_msg("Got the access token for login.gov")
+        user_email, user_uuid = sign_in._get_user_email_and_uuid(access_token)
+        debug_msg(
+            f"Got the user_email {user_email} and user_uuid {user_uuid} from login.gov"
+        )
+        invite_data = state.encode("utf8")
+        invite_data = base64.b64decode(invite_data)
+        invite_data = json.loads(invite_data)
+        debug_msg(f"final state {invite_data}")
+        invited_user_id = invite_data["invited_user_id"]
+        invited_user_email_address = get_invited_user_email_address(invited_user_id)
+        debug_msg(f"email address from the invite_date is {invited_user_email_address}")
+        check_invited_user_email_address_matches_expected(
+            user_email, invited_user_email_address
+        )
+
+        invited_user_accept_invite(invited_user_id)
+        debug_msg(
+            f"accepted invite user {invited_user_email_address} to service {invite_data['service_id']}"
+        )
+        # We need to avoid taking a second trip through the login.gov code because we cannot pull the
+        # access token twice.  So once we retrieve these values, let's park them in redis for 15 minutes
+        put_invite_data_in_redis(
+            state, invite_data, user_email, user_uuid, invited_user_email_address
+        )
+
     form = SetupUserProfileForm()
 
-    if form.validate_on_submit():
-        # start login.gov
-        code = request.args.get("code")
-        state = request.args.get("state")
-        login_gov_error = request.args.get("error")
-        if code and state:
-            access_token = sign_in._get_access_token(code, state)
-            user_email, user_uuid = sign_in._get_user_email_and_uuid(access_token)
-            redirect_url = request.args.get("next")
-
-        elif login_gov_error:
-            current_app.logger.error(f"login.gov error: {login_gov_error}")
-            raise Exception(f"Could not login with login.gov {login_gov_error}")
-        # end login.gov
-
-        # create the user
-        # TODO we have to provide something for password until that column goes away
-        # TODO ideally we would set the user's preferred timezone here as well
+    if (
+        form.validate_on_submit()
+        and redis_client.get(f"invitedata-{state}") is not None
+    ):
+        invite_data, user_email, user_uuid, invited_user_email_address = (
+            get_invite_data_from_redis(state)
+        )
 
+        # create or update the user
         user = user_api_client.get_user_by_uuid_or_email(user_uuid, user_email)
         if user is None:
             user = User.register(
@@ -170,9 +211,63 @@ def set_up_your_profile():
                 password=str(uuid.uuid4()),
                 auth_type="sms_auth",
             )
+            debug_msg(f"registered user {form.name.data} with email {user_email}")
+        else:
+            user.update(mobile_number=form.mobile_number.data, name=form.name.data)
+            debug_msg(f"updated user {form.name.data}")
 
         # activate the user
         user = user_api_client.get_user_by_uuid_or_email(user_uuid, user_email)
         activate_user(user["id"])
-        return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url))
+        debug_msg("activated user")
+        usr = User.from_id(user["id"])
+        usr.add_to_service(
+            invite_data["service_id"],
+            invite_data["permissions"],
+            invite_data["folder_permissions"],
+            invite_data["from_user_id"],
+        )
+        debug_msg(
+            f"Added user {usr.email_address} to service {invite_data['service_id']}"
+        )
+        return redirect(url_for("main.show_accounts_or_dashboard"))
+
+    elif login_gov_error:
+        current_app.logger.error(f"login.gov error: {login_gov_error}")
+        abort(403)
+
+    # we take two trips through this method, but should only hit this
+    # line on the first trip.  On the second trip, we should get redirected
+    # to the accounts page because we have successfully registered.
     return render_template("views/set-up-your-profile.html", form=form)
+
+
+def get_invited_user_email_address(invited_user_id):
+    # InvitedUser is an unhashable type and hard to mock in tests
+    # so this convenience method is a workaround for that
+    invited_user = InvitedUser.by_id(invited_user_id)
+    return invited_user.email_address
+
+
+def invited_user_accept_invite(invited_user_id):
+    invited_user = InvitedUser.by_id(invited_user_id)
+
+    if invited_user.status == "expired":
+        current_app.logger.error("User invitation has expired")
+        flash(
+            "Your invitation has expired; please contact the person who invited you for additional help."
+        )
+        abort(401)
+
+    if invited_user.status == "cancelled":
+        current_app.logger.error("User invitation has been cancelled")
+        flash(
+            "Your invitation is no longer valid; please contact the person who invited you for additional help."
+        )
+        abort(401)
+
+    invited_user.accept_invite()
+
+
+def debug_msg(msg):
+    current_app.logger.debug(hilite(msg))

app/main/views/send.py

@@ -1,9 +1,19 @@
+from pipes import quote
 import time
 import uuid
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
-from flask import abort, flash, redirect, render_template, request, session, url_for
+from flask import (
+    abort,
+    current_app,
+    flash,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
 from flask_login import current_user
 from markupsafe import Markup
 from notifications_python_client.errors import HTTPError
@@ -31,7 +41,12 @@ from app.s3_client.s3_csv_client import (
     s3upload,
     set_metadata_on_csv_upload,
 )
-from app.utils import PermanentRedirect, should_skip_template_page, unicode_truncate
+from app.utils import (
+    PermanentRedirect,
+    hilite,
+    should_skip_template_page,
+    unicode_truncate,
+)
 from app.utils.csv import Spreadsheet, get_errors_for_csv
 from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
@@ -79,6 +94,7 @@ def get_example_csv_rows(template, use_example_as_example=True, submitted_fields
 )
 @user_has_permissions("send_messages", restrict_admin_usage=True)
 def send_messages(service_id, template_id):
+    print(hilite("top of send_messages"))
     notification_count = service_api_client.get_notification_count(service_id)
     remaining_messages = current_service.message_limit - notification_count
 
@@ -470,7 +486,8 @@ def send_one_off_step(service_id, template_id, step_index):
     )
 
 
-def _check_messages(service_id, template_id, upload_id, preview_row):
+def _check_messages(service_id, template_id, upload_id, preview_row, **kwargs):
+    print(hilite("top of _check_messages"))
     try:
         # The happy path is that the job doesn’t already exist, so the
         # API will return a 404 and the client will raise HTTPError.
@@ -510,11 +527,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row):
         show_recipient=False,
         email_reply_to=email_reply_to,
         sms_sender=sms_sender,
-    )
-    simplifed_template = get_template(
-        db_template,
-        current_service,
-        show_recipient=False,
+        **kwargs,
     )
 
     allow_list = []
@@ -535,7 +548,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row):
         allow_list = None
     recipients = RecipientCSV(
         contents,
-        template=template or simplifed_template,
+        template=template,
         max_initial_rows_shown=50,
         max_errors_shown=50,
         guestlist=allow_list,
@@ -569,9 +582,6 @@ def _check_messages(service_id, template_id, upload_id, preview_row):
 
     if preview_row < len(recipients) + 2:
         template.values = recipients[preview_row - 2].recipient_and_personalisation
-        simplifed_template.values = recipients[
-            preview_row - 2
-        ].recipient_and_personalisation
     elif preview_row > 2:
         abort(404)
 
@@ -599,7 +609,6 @@ def _check_messages(service_id, template_id, upload_id, preview_row):
             service_id, template.id, db_template["version"], original_file_name
         ),
         template_id=template_id,
-        simplifed_template=simplifed_template,
     )
 
 
@@ -614,6 +623,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row):
 @user_has_permissions("send_messages", restrict_admin_usage=True)
 def check_messages(service_id, template_id, upload_id, row_index=2):
     data = _check_messages(service_id, template_id, upload_id, row_index)
+    print(hilite(f"data after checking messages: \n {data}"))
     data["allowed_file_extensions"] = Spreadsheet.ALLOWED_FILE_EXTENSIONS
 
     if (
@@ -658,7 +668,9 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
 @user_has_permissions("send_messages", restrict_admin_usage=True)
 def preview_job(service_id, template_id, upload_id, row_index=2):
     session["scheduled_for"] = request.form.get("scheduled_for", "")
-    data = _check_messages(service_id, template_id, upload_id, row_index)
+    data = _check_messages(
+        service_id, template_id, upload_id, row_index, force_hide_sender=True
+    )
 
     return render_template(
         "views/check/preview.html",
@@ -825,11 +837,11 @@ def send_one_off_to_myself(service_id, template_id):
 def check_notification(service_id, template_id):
     return render_template(
         "views/notifications/check.html",
-        **_check_notification(service_id, template_id),
+        **_check_notification(service_id, template_id, show_recipient=True),
     )
 
 
-def _check_notification(service_id, template_id, exception=None):
+def _check_notification(service_id, template_id, exception=None, **kwargs):
     db_template = current_service.get_template_with_user_permission_or_403(
         template_id, current_user
     )
@@ -842,13 +854,9 @@ def _check_notification(service_id, template_id, exception=None):
     template = get_template(
         db_template,
         current_service,
-        show_recipient=True,
         email_reply_to=email_reply_to,
         sms_sender=sms_sender,
-    )
-    simplifed_template = get_template(
-        db_template,
-        current_service,
+        **kwargs,
     )
     placeholders = fields_to_fill_in(template)
 
@@ -874,7 +882,6 @@ def _check_notification(service_id, template_id, exception=None):
         back_link_from_preview=back_link_from_preview,
         choose_time_form=choose_time_form,
         **(get_template_error_dict(exception) if exception else {}),
-        simplifed_template=simplifed_template,
     )
 
 
@@ -924,7 +931,9 @@ def preview_notification(service_id, template_id):
 
     return render_template(
         "views/notifications/preview.html",
-        **_check_notification(service_id, template_id),
+        **_check_notification(
+            service_id, template_id, show_recipient=False, force_hide_sender=True
+        ),
         scheduled_for=session["scheduled_for"],
         recipient=recipient,
     )
@@ -951,15 +960,31 @@ def send_notification(service_id, template_id):
     values = []
     for k, v in session["placeholders"].items():
         keys.append(k)
-        values.append(v)
+        print(hilite(f"value: {v} type: {type(v)}"))
+        value_quotes = '"' + v + '"'
+        print(hilite(f"value with quotes: {value_quotes}"))
+        values.append(value_quotes)
 
     data = ",".join(keys)
     vals = ",".join(values)
     data = f"{data}\r\n{vals}"
 
-    filename = f"one-off-{current_user.name}-{uuid.uuid4()}.csv"
+    filename = (
+        f"one-off-{uuid.uuid4()}.csv"  # {current_user.name} removed from filename
+    )
     my_data = {"filename": filename, "template_id": template_id, "data": data}
     upload_id = s3upload(service_id, my_data)
+
+    # To debug messages that the user reports have not been sent, we log
+    # the csv filename and the job id.  The user will give us the file name,
+    # so we can search on that to obtain the job id, which we can use elsewhere
+    # on the API side to find out what happens to the message.
+    current_app.logger.info(
+        hilite(
+            f"One-off file: {filename} job_id: {upload_id} s3 location: service-{service_id}-notify/{upload_id}.csv"
+        )
+    )
+
     form = CsvUploadForm()
     form.file.data = my_data
     form.file.name = filename
@@ -1009,14 +1034,17 @@ def send_notification(service_id, template_id):
                 job_id=upload_id,
             )
         )
-
+    total = notifications["total"]
+    current_app.logger.info(
+        hilite(
+            f"job_id: {upload_id} has notifications: {total} and attempts: {attempts}"
+        )
+    )
     return redirect(
         url_for(
             ".view_job",
             service_id=service_id,
             job_id=upload_id,
-            from_job=upload_id,
-            notification_id=notifications["notifications"][0]["id"],
             # used to show the final step of the tour (help=3) or not show
             # a back link on a just sent one off notification (help=0)
             help=request.args.get("help"),

app/main/views/sign_in.py

@@ -5,26 +5,26 @@ import uuid
 import jwt
 import requests
 from flask import (
-    Markup,
+    Response,
     abort,
     current_app,
     flash,
     redirect,
     render_template,
     request,
-    session,
     url_for,
 )
 from flask_login import current_user
 
 from app import login_manager, user_api_client
 from app.main import main
-from app.main.forms import LoginForm
 from app.main.views.index import error
 from app.main.views.verify import activate_user
-from app.models.user import InvitedUser, User
+from app.models.user import User
 from app.utils import hide_from_search_engines
 from app.utils.login import is_safe_redirect_url
+from app.utils.time import is_less_than_days_ago
+from app.utils.user import is_gov_user
 from notifications_utils.url_safe_token import generate_token
 
 
@@ -63,6 +63,10 @@ def _get_access_token(code, state):
     url = f"{base_url}{cli_assert}&{cli_assert_type}&{code_param}&grant_type=authorization_code"
     headers = {"Authorization": "Bearer %s" % token}
     response = requests.post(url, headers=headers)
+    if response.json().get("access_token") is None:
+        # Capture the response json here so it hopefully shows up in error reports
+        current_app.logger.error(f"Error when getting access token {response.json()}")
+        raise KeyError(f"'access_token' {response.json()}")
     access_token = response.json()["access_token"]
     return access_token
 
@@ -84,98 +88,85 @@ def _do_login_dot_gov():
     code = request.args.get("code")
     state = request.args.get("state")
     login_gov_error = request.args.get("error")
-    if code and state:
-        access_token = _get_access_token(code, state)
-        user_email, user_uuid = _get_user_email_and_uuid(access_token)
-        redirect_url = request.args.get("next")
+
+    if login_gov_error:
+        current_app.logger.error(f"login.gov error: {login_gov_error}")
+        raise Exception(f"Could not login with login.gov {login_gov_error}")
+    elif code and state:
 
         # activate the user
         try:
+            access_token = _get_access_token(code, state)
+            user_email, user_uuid = _get_user_email_and_uuid(access_token)
+            if not is_gov_user(user_email):
+                current_app.logger.error(
+                    "invited user has a non-government email address."
+                )
+                flash("You must use a government email address.")
+                abort(403)
+            redirect_url = request.args.get("next")
             user = user_api_client.get_user_by_uuid_or_email(user_uuid, user_email)
-            activate_user(user["id"])
+
+            # Check if the email needs to be revalidated
+            is_fresh_email = is_less_than_days_ago(
+                user["email_access_validated_at"], 90
+            )
+            if not is_fresh_email:
+                return verify_email(user, redirect_url)
+
+            usr = User.from_email_address(user["email_address"])
+            activate_user(usr.id)
         except BaseException as be:  # noqa B036
             current_app.logger.error(be)
             error(401)
-
         return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url))
 
-    elif login_gov_error:
-        current_app.logger.error(f"login.gov error: {login_gov_error}")
-        raise Exception(f"Could not login with login.gov {login_gov_error}")
     # end login.gov
 
 
+def verify_email(user, redirect_url):
+    user_api_client.send_verify_code(user["id"], "email", None, redirect_url)
+    title = "Email resent" if request.args.get("email_resent") else "Check your email"
+    redirect_url = request.args.get("next")
+    return render_template(
+        "views/re-validate-email-sent.html", title=title, redirect_url=redirect_url
+    )
+
+
+def _handle_e2e_tests(redirect_url):
+    current_app.logger.warning("E2E TESTS ARE ENABLED.")
+    current_app.logger.warning(
+        "If you are getting a 404 on signin, comment out E2E vars in .env file!"
+    )
+    user = user_api_client.get_user_by_email(os.getenv("NOTIFY_E2E_TEST_EMAIL"))
+    activate_user(user["id"])
+    return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url))
+
+
 @main.route("/sign-in", methods=(["GET", "POST"]))
 @hide_from_search_engines
 def sign_in():
-    _do_login_dot_gov()
+    # If we have to revalidated the email, send the message
+    # via email and redirect to the "verify your email page"
+    # and don't proceed further with login
+    email_verify_template = _do_login_dot_gov()
+    if (
+        email_verify_template
+        and not isinstance(email_verify_template, Response)
+        and "Check your email" in email_verify_template
+    ):
+        return email_verify_template
+
     redirect_url = request.args.get("next")
 
     if os.getenv("NOTIFY_E2E_TEST_EMAIL"):
-        current_app.logger.warning("E2E TESTS ARE ENABLED.")
-        current_app.logger.warning(
-            "If you are getting a 404 on signin, comment out E2E vars in .env file!"
-        )
-        user = user_api_client.get_user_by_email(os.getenv("NOTIFY_E2E_TEST_EMAIL"))
-        activate_user(user["id"])
-        return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url))
+        return _handle_e2e_tests(redirect_url)
 
-    current_app.logger.info(f"current user is {current_user}")
     if current_user and current_user.is_authenticated:
         if redirect_url and is_safe_redirect_url(redirect_url):
             return redirect(redirect_url)
         return redirect(url_for("main.show_accounts_or_dashboard"))
 
-    form = LoginForm()
-    current_app.logger.info("Got the login form")
-    password_reset_url = url_for(".forgot_password", next=request.args.get("next"))
-
-    if form.validate_on_submit():
-        user = User.from_email_address_and_password_or_none(
-            form.email_address.data, form.password.data
-        )
-
-        if user:
-            # add user to session to mark us as in the process of signing the user in
-            session["user_details"] = {"email": user.email_address, "id": user.id}
-
-            if user.state == "pending":
-                return redirect(
-                    url_for("main.resend_email_verification", next=redirect_url)
-                )
-
-            if user.is_active:
-                if session.get("invited_user_id"):
-                    invited_user = InvitedUser.from_session()
-                    if user.email_address.lower() != invited_user.email_address.lower():
-                        flash("You cannot accept an invite for another person.")
-                        session.pop("invited_user_id", None)
-                        abort(403)
-                    else:
-                        invited_user.accept_invite()
-
-                user.send_login_code()
-
-                if user.sms_auth:
-                    return redirect(url_for(".two_factor_sms", next=redirect_url))
-
-                if user.email_auth:
-                    return redirect(
-                        url_for(".two_factor_email_sent", next=redirect_url)
-                    )
-
-        # Vague error message for login in case of user not known, locked, inactive or password not verified
-        flash(
-            Markup(
-                (
-                    f"The email address or password you entered is incorrect."
-                    f"&ensp;<a href={password_reset_url} class='usa-link'>Forgot your password?</a>"
-                )
-            )
-        )
-
-    other_device = current_user.logged_in_elsewhere()
-
     token = generate_token(
         str(request.remote_addr),
         current_app.config["SECRET_KEY"],
@@ -186,14 +177,9 @@ def sign_in():
     if url is not None:
         url = url.replace("NONCE", token)
         url = url.replace("STATE", token)
-
     return render_template(
         "views/signin.html",
-        form=form,
         again=bool(redirect_url),
-        other_device=other_device,
-        login_gov_enabled=True,
-        password_reset_url=password_reset_url,
         initial_signin_url=url,
     )
 

app/main/views/sub_navigation_dictionaries.py

@@ -32,7 +32,7 @@ def using_notify_nav():
             "link": "main.trial_mode_new",
         },
         {
-            "name": "Pricing",
+            "name": "Tracking usage",
             "link": "main.pricing",
         },
         {

app/main/views/tour.py

@@ -195,10 +195,10 @@ def check_tour_notification(service_id, template_id):
     )
 
     return render_template(
-        "views/notifications/check.html",
+        "views/notifications/preview.html",
         template=template,
         back_link=back_link,
-        help="2",
+        help="3",
     )
 
 

app/main/views/verify.py

@@ -4,10 +4,9 @@ from flask import abort, current_app, flash, redirect, render_template, session,
 from itsdangerous import SignatureExpired
 
 from app import user_api_client
-from app.extensions import redis_client
 from app.main import main
 from app.main.forms import TwoFactorForm
-from app.models.user import InvitedOrgUser, InvitedUser, User
+from app.models.user import User
 from app.utils.login import redirect_to_sign_in
 from notifications_utils.url_safe_token import check_token
 
@@ -66,59 +65,19 @@ def verify_email(token):
 def activate_user(user_id):
     user = User.from_id(user_id)
 
-    # This is the login.gov path
-    login_gov_invite_data = redis_client.get(f"service-invite-{user.email_address}")
-    if login_gov_invite_data:
-        login_gov_invite_data = json.loads(login_gov_invite_data.decode("utf8"))
-
-    # This is the deprecated path for organization invites where we get id from session
-    session["current_session_id"] = user.current_session_id
-    organization_id = session.get("organization_id")
-
-    activated_user = user.activate()
-    activated_user.login()
-
-    # TODO when login.gov is mandatory, get rid of the if clause, it is deprecated.
-    invited_user = InvitedUser.from_session()
-    if invited_user:
-        service_id = _add_invited_user_to_service(invited_user)
-        return redirect(url_for("main.service_dashboard", service_id=service_id))
-    elif login_gov_invite_data:
-        service_id = login_gov_invite_data["service_id"]
-
-        user.add_to_service(
-            service_id,
-            login_gov_invite_data["permissions"],
-            login_gov_invite_data["folder_permissions"],
-            login_gov_invite_data["from_user_id"],
-        )
-        return redirect(url_for("main.service_dashboard", service_id=service_id))
-
-    # TODO when login.gov is mandatory, git rid of the if clause, it is deprecated.
-    invited_org_user = InvitedOrgUser.from_session()
-    if invited_org_user:
-        user_api_client.add_user_to_organization(invited_org_user.organization, user_id)
-    elif redis_client.get(f"organization-invite-{user.email_address}"):
-        organization_id = redis_client.raw_get(
-            f"organization-invite-{user.email_address}"
-        )
-        user_api_client.add_user_to_organization(
-            organization_id.decode("utf8"), user_id
-        )
+    # TODO add org invites back in the new way
+    # organization_id = redis_client.get(
+    #   f"organization-invite-{user.email_address}"
+    # )
+    # user_api_client.add_user_to_organization(
+    #   organization_id.decode("utf8"), user_id
+    # )
+    organization_id = None
 
     if organization_id:
         return redirect(url_for("main.organization_dashboard", org_id=organization_id))
     else:
+        activated_user = user.activate()
+        activated_user.login()
+
         return redirect(url_for("main.add_service", first="first"))
-
-
-def _add_invited_user_to_service(invitation):
-    user = User.from_id(session["user_id"])
-    service_id = invitation.service
-    user.add_to_service(
-        service_id,
-        invitation.permissions,
-        invitation.folder_permissions,
-        invitation.from_user.id,
-    )
-    return service_id

app/models/service.py

@@ -390,7 +390,7 @@ class Service(JSONModel, SortByNameMixin):
     def get_data_retention_item(self, id):
         return next((dr for dr in self.data_retention if dr["id"] == id), None)
 
-    def get_days_of_retention(self, notification_type):
+    def get_days_of_retention(self, notification_type, number_of_days):
         return next(
             (
                 dr
@@ -398,7 +398,10 @@ class Service(JSONModel, SortByNameMixin):
                 if dr["notification_type"] == notification_type
             ),
             {},
-        ).get("days_of_retention", current_app.config["ACTIVITY_STATS_LIMIT_DAYS"])
+        ).get(
+            "days_of_retention",
+            current_app.config["ACTIVITY_STATS_LIMIT_DAYS"].get(number_of_days),
+        )
 
     @cached_property
     def organization(self):

app/models/user.py

@@ -140,9 +140,6 @@ class User(JSONModel, UserMixin):
             set_by_id=set_by_id,
         )
 
-    def logged_in_elsewhere(self):
-        return session.get("current_session_id") != self.current_session_id
-
     def activate(self):
         if self.is_pending:
             user_data = user_api_client.activate_user(self.id)
@@ -196,7 +193,7 @@ class User(JSONModel, UserMixin):
 
     @property
     def is_authenticated(self):
-        return not self.logged_in_elsewhere() and super(User, self).is_authenticated
+        return super(User, self).is_authenticated
 
     @property
     def platform_admin(self):
@@ -674,10 +671,6 @@ class InvitedOrgUser(JSONModel):
 
 
 class AnonymousUser(AnonymousUserMixin):
-    # set the anonymous user so that if a new browser hits us we don't error http://stackoverflow.com/a/19275188
-
-    def logged_in_elsewhere(self):
-        return False
 
     @property
     def default_organization(self):

app/notify_client/service_api_client.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 
 from app.extensions import redis_client
 from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
@@ -43,6 +43,28 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             params={"limit_days": limit_days},
         )["data"]
 
+    def get_service_notification_statistics_by_day(
+        self, service_id, start_date=None, days=None
+    ):
+        if start_date is None:
+            start_date = datetime.now().strftime("%Y-%m-%d")
+
+        return self.get(
+            "/service/{0}/statistics/{1}/{2}".format(service_id, start_date, days),
+        )["data"]
+
+    def get_user_service_notification_statistics_by_day(
+        self, service_id, user_id, start_date=None, days=None
+    ):
+        if start_date is None:
+            start_date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+        return self.get(
+            "/service/{0}/statistics/user/{1}/{2}/{3}".format(
+                service_id, user_id, start_date, days
+            ),
+        )["data"]
+
     def get_services(self, params_dict=None):
         """
         Retrieve a list of services.
@@ -497,5 +519,18 @@ class ServiceAPIClient(NotifyAdminAPIClient):
     def get_global_notification_count(self, service_id):
         return self.get("/service/{}/notification-count".format(service_id))
 
+    def get_service_invite_data(self, redis_key):
+        """
+        Retrieve service invite_data.
+        """
+        return self.get("/service/invite/redis/{0}".format(redis_key))
+
 
 service_api_client = ServiceAPIClient()
+
+
+# TODO, if we try to call get_service_invite_data directly
+# from verify, app complains the method is not defined
+# If we wrap it like this, the app can find it.
+def retrieve_service_invite_data(redis_key):
+    return service_api_client.get_service_invite_data(redis_key)

app/s3_client/__init__.py

@@ -1,3 +1,5 @@
+import os
+
 import botocore
 from boto3 import Session
 from botocore.config import Config
@@ -29,6 +31,17 @@ def get_s3_object(
     )
     s3 = session.resource("s3", config=AWS_CLIENT_CONFIG)
     obj = s3.Object(bucket_name, filename)
+    # This 'proves' that use of moto in the relevant tests in test_send.py
+    # mocks everything related to S3.  What you will see in the logs is:
+    # Exception: CREATED AT <MagicMock name='resource().Bucket().creation_date' id='4665562448'>
+    #
+    # raise Exception(f"CREATED AT {_s3.Bucket(bucket_name).creation_date}")
+    if os.getenv("NOTIFY_ENVIRONMENT") == "test":
+        teststr = str(s3.Bucket(bucket_name).creation_date).lower()
+        if "magicmock" not in teststr:
+            raise Exception(
+                "Test is not mocked, use @mock_aws or the relevant mocker.patch to avoid accessing S3"
+            )
     return obj
 
 

app/s3_client/s3_csv_client.py

@@ -28,6 +28,7 @@ def get_csv_upload(service_id, upload_id):
 
 
 def s3upload(service_id, filedata):
+
     upload_id = str(uuid.uuid4())
     bucket_name, file_location, access_key, secret_key, region = get_csv_location(
         service_id, upload_id

app/templates/base.html

@@ -1,10 +1,17 @@
-{% from "../components/banner.html" import banner %}
-{% from "../components/components/skip-link/macro.njk" import usaSkipLink -%}
-
+{% from "components/banner.html" import banner %}
+{% from "components/components/skip-link/macro.njk" import usaSkipLink -%}
+{% from "components/sub-navigation.html" import sub_navigation %}
 
 <!DOCTYPE html>
 <html lang="{{ htmlLang | default('en') }}" class="{{ htmlClasses }}">
-  {% include "new/components/head.html" %}
+  <head>
+    <title>
+      {% block pageTitle %}
+        {% block per_page_title %}{% endblock %} – Notify.gov
+      {% endblock %}
+    </title>
+    {% include "new/components/head.html" %}
+  </head>
   <body class="usa-template__body {{ bodyClasses }}">
     <script nonce="{{ csp_nonce() }}">document.body.className = ((document.body.className) ? document.body.className + ' js-enabled' : 'js-enabled');</script>
     {% block bodyStart %}
@@ -23,11 +30,10 @@
       }) }}
     {% endblock %}
 
+
     {% block header %}
-      {% if current_user.is_authenticated %}
-        {% include 'new/components/usa_banner.html' %}
-        {% include 'new/components/header.html' %}
-      {% endif %}
+      {% include 'new/components/usa_banner.html' %}
+      {% include 'new/components/header.html' %}
     {% endblock %}
 
     {% block main %}
@@ -36,18 +42,17 @@
         {% block backLink %}{% endblock %}
       {% endblock %}
       {% block mainClasses %}
-      <!-- notes set mainClasses = "margin-top-5 padding-bottom-5" where withoutnav_template was used and maybe templates that are using content_template -->
+      {% set mainClasses = "margin-top-5 padding-bottom-5" %}
       <main class="{{ mainClasses }}" id="main-content" role="main">
       {% endblock %}
         {% block content %}
           {% block flash_messages %}
-            <!-- flash_message.html was from the withoutnav_template and is only included on child templates that was using withoutnav_template. Now, we can add in flash_message blocks and include 'flash_messages.html' to child templates that was using withoutnav_template. This will help to eliminate the use of a whole other parent template. -->
+            {% include 'new/components/flash_messages.html' %}
           {% endblock %}
           {% block maincolumn_content %}
-            {% block fromContentTemplatetwoColumnGrid %}
               <div class="grid-row">
                 {% if navigation_links %}
-                  <div class="tablet:grid-col-2">
+                  <div class="tablet:grid-col-2 margin-bottom-4">
                     {{ sub_navigation(navigation_links) }}
                   </div>
                   <div class="tablet:grid-col-10 padding-left-4 usa-prose site-prose">
@@ -57,8 +62,6 @@
                 {% block content_column_content %}{% endblock %}
                 </div>
               </div>
-              <!-- content_column_content block is from the content_template.html. We do not need this template. We can consolidate and move it to the base.html template. We can call on this block where child templates were using the content_template.html  -->
-            {% endblock %}
           {% endblock %}
         {% endblock %}
       </main>

app/templates/error/401.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}You’re not authorised to see this page{% endblock %}
 {% block maincolumn_content %}
   <div class="grid-row">

app/templates/error/403.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}You’re not allowed to see this page{% endblock %}
 {% block maincolumn_content %}
     <div class="grid-row">

app/templates/error/404.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}Page not found{% endblock %}
 {% block maincolumn_content %}
     <div class="grid-row">

app/templates/error/410.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}Page not found{% endblock %}
 {% block maincolumn_content %}
     <div class="grid-row">

app/templates/error/500.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}Sorry, there’s a problem with the service{% endblock %}
 {% block maincolumn_content %}
   <div class="grid-row">
@@ -7,7 +7,7 @@
         Sorry, we can't deliver what you asked for right now.
       </h1>
       <p class="usa-body">
-        Please try again later or <a class="usa-link" href="mailto:notify-support@gsa.gov"></a>email us</a> for more information.</p>
+        Please try again later or <a class="usa-link" href="mailto:notify-support@gsa.gov">email us</a> for more information.</p>
       </p>
     </div>
   </div>

app/templates/new/components/head.html

@@ -1,6 +1,4 @@
-<head>
     <meta charset="utf-8" />
-    <title>{% block pageTitle %}Notify.gov{% endblock %}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta name="description" content="Reach people where they are with a text message platform built just for government">
     <meta name="theme-color" media="(prefers-color-scheme: light)" content="f0f0f0" />
@@ -12,7 +10,7 @@
     {# Ensure that older IE versions always render with the correct rendering engine #}
     <meta http-equiv="X-UA-Compatible" content="IE=edge" />
 
-    {% block headIcons %}
+
     <link rel="shortcut icon" href="{{ assetPath | default('/assets') }}/images/favicon.ico" />
     <link rel="icon" type="image/png" sizes="32x32" href="{{ assetPath | default('/assets') }}/images/favicon-32x32.png" />
     <link rel="icon" type="image/png" sizes="16x16" href="{{ assetPath | default('/assets') }}/images/favicon-16x16.png" />
@@ -22,25 +20,18 @@
     <meta name="msapplication-TileColor" content="#da532c">
     <link href="{{ assetPath | default('/assets') }}/images/notify-dark-favicon.png" rel="icon" media="(prefers-color-scheme: dark)">
     <meta name="theme-color" content="#ffffff">
-    {% endblock %}
+
 
     <link rel="stylesheet" media="screen" href="{{ asset_url('css/styles.css') }}" />
     {% block extra_stylesheets %}{% endblock %}
 
-    {% block meta_format_detection %}
     <meta name="format-detection" content="telephone=no">
-    {% endblock %}
-    {% block og_image %}
     <meta property="og:title" content="Notify.gov">
     <meta property="og:description" content="Reach people where they are with a text message platform built just for government">
     <meta property="og:image" content="/static/images/notify-og-image.png">
-    {% endblock %}
     {# google #}
     <script type="text/javascript" src="{{ asset_url('js/gtm_head.js') }}"></script>
     <meta name="google-site-verification" content="niWnSqImOWz6mVQTYqNb5tFK8HaKSB4b3ED4Z9gtUQ0" />
     {% if g.hide_from_search_engines %}
     <meta name="robots" content="noindex" />
     {% endif %}
-
-    {% block head %}{% endblock %}
-</head>

app/templates/new/components/header.html

@@ -1,25 +1,28 @@
 {# setting navigation and secondarynavigation #}
-{% set navigation = [
-  {"href": url_for("main.show_accounts_or_dashboard"), "text": "Current service", "active": header_navigation.is_selected('accounts-or-dashboard')},
-  {"href": url_for('main.get_started'), "text": "Using Notify", "active": header_navigation.is_selected('using_notify')},
-  {"href": url_for('main.features'), "text": "Features", "active": header_navigation.is_selected('features')},
-  {"href": url_for('main.support'), "text": "Contact us", "active": header_navigation.is_selected('support')}
-] %}
-
-{% if current_user.platform_admin %}
-  {% set navigation = navigation + [{"href": url_for('main.platform_admin_splash_page'), "text": "Platform admin", "active": header_navigation.is_selected('platform-admin')}] %}
-{% else %}
-  {% set navigation = navigation + [{"href": url_for('main.user_profile'), "text": "User profile", "active": header_navigation.is_selected('user-profile')}] %}
-{% endif %}
-
-{% if current_service %}
-  {% set secondaryNavigation = [
-      {"href": url_for('main.service_settings', service_id=current_service.id), "text": "Settings", "active": secondary_navigation.is_selected('settings')},
-      {"href": url_for('main.sign_out'), "text": "Sign out"}
+{% if current_user.is_authenticated %}
+  {% set navigation = [
+    {"href": url_for("main.show_accounts_or_dashboard"), "text": "Current service", "active": header_navigation.is_selected('accounts-or-dashboard')},
+    {"href": url_for('main.get_started'), "text": "Using Notify", "active": header_navigation.is_selected('using_notify')},
+    {"href": url_for('main.features'), "text": "Features", "active": header_navigation.is_selected('features')},
+    {"href": url_for('main.support'), "text": "Contact us", "active": header_navigation.is_selected('support')}
   ] %}
-{% else %}
-  {% set secondaryNavigation = [{"href": url_for('main.sign_out'), "text": "Sign out"}] %}
+
+  {% if current_user.platform_admin %}
+    {% set navigation = navigation + [{"href": url_for('main.platform_admin_splash_page'), "text": "Platform admin", "active": header_navigation.is_selected('platform-admin')}] %}
+  {% else %}
+    {% set navigation = navigation + [{"href": url_for('main.user_profile'), "text": "User profile", "active": header_navigation.is_selected('user-profile')}] %}
+  {% endif %}
+
+  {% if current_service %}
+    {% set secondaryNavigation = [
+        {"href": url_for('main.service_settings', service_id=current_service.id), "text": "Settings", "active": secondary_navigation.is_selected('settings')},
+        {"href": url_for('main.sign_out'), "text": "Sign out"}
+    ] %}
+  {% else %}
+    {% set secondaryNavigation = [{"href": url_for('main.sign_out'), "text": "Sign out"}] %}
+  {% endif %}
 {% endif %}
+
 {# usa header #}
 <header class="usa-header usa-header--extended">
   <div class="usa-nav-container">

app/templates/new/components/main_nav.html

@@ -0,0 +1,30 @@
+{% if help %}
+  {% include 'partials/tour.html' %}
+{% else %}
+<nav class="nav margin-bottom-4">
+  <a class="usa-button margin-top-1 margin-bottom-5 width-full"
+    href="{{ url_for('.choose_template', service_id=current_service.id) }}">Send messages</a>
+  <ul class="usa-sidenav">
+  {% if current_user.has_permissions() %}
+    {% if current_user.has_permissions('view_activity') %}
+      <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('dashboard') }}" href="{{ url_for('.service_dashboard', service_id=current_service.id) }}">Dashboard</a></li>
+    {% endif %}
+    {% if not current_user.has_permissions('view_activity') %}
+      <li class="usa-sidenav__item"><a class="{{ casework_navigation.is_selected('sent-messages') }}" href="{{ url_for('.view_notifications', service_id=current_service.id, status='sending,delivered,failed') }}">Sent messages</a></li>
+    {% endif %}
+    {% if current_user.has_permissions('manage_service', allow_org_user=True) %}
+      {# <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('usage') }}" href="{{ url_for('.usage', service_id=current_service.id) }}">Usage</a></li> #}
+    {% endif %}
+    <!-- {% if current_user.has_permissions('manage_api_keys', 'manage_service') %}
+      <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('settings') }}" href="{{ url_for('.service_settings', service_id=current_service.id) }}">Settings</a></li>
+    {% endif %} -->
+    {% if current_user.has_permissions('manage_api_keys') %}
+      <!-- <li><a class="usa-link{{ main_navigation.is_selected('api-integration') }}" href="{{ url_for('.api_integration', service_id=current_service.id) }}">API integration</a></li> -->
+    {% endif %}
+  {% elif current_user.has_permissions(allow_org_user=True) %}
+    <li class="usa-sidenav__item"><a class="usa-link{{ main_navigation.is_selected('usage') }}" href="{{ url_for('.usage', service_id=current_service.id) }}">Usage</a></li>
+    <li class="usa-sidenav__item"><a class="usa-link{{ main_navigation.is_selected('team-members') }}" href="{{ url_for('.manage_users', service_id=current_service.id) }}">Team members</a></li>
+  {% endif %}
+  </ul>
+</nav>
+{% endif %}

app/templates/new/components/org_nav.html

@@ -0,0 +1,11 @@
+<nav class="nav margin-bottom-4">
+  <ul class="usa-sidenav">
+    <li class="usa-sidenav__item"><a class="usa-link{{ org_navigation.is_selected('dashboard') }}" href="{{ url_for('.organization_dashboard', org_id=current_org.id) }}">Usage</a></li>
+    <li class="usa-sidenav__item"><a class="usa-link{{ org_navigation.is_selected('team-members') }}" href="{{ url_for('.manage_org_users', org_id=current_org.id) }}">Team members</a></li>
+    {% if current_user.platform_admin %}
+    <li class="usa-sidenav__item"><a class="usa-link{{ org_navigation.is_selected('settings') }}" href="{{ url_for('.organization_settings', org_id=current_org.id) }}">Settings</a></li>
+    <li class="usa-sidenav__item"><a class="usa-link{{ org_navigation.is_selected('trial-services') }}" href="{{ url_for('.organization_trial_mode_services', org_id=current_org.id) }}">Trial mode services</a></li>
+    <li class="usa-sidenav__item"><a class="usa-link{{ org_navigation.is_selected('billing') }}" href="{{ url_for('.organization_billing', org_id=current_org.id) }}">Billing</a></li>
+    {% endif %}
+  </ul>
+</nav>

app/templates/new/components/org_nav_breadcrumb.html

@@ -0,0 +1,11 @@
+<nav class="navigation-service usa-breadcrumb">
+    <ol class="usa-breadcrumb__list">
+      <li class="usa-breadcrumb__list-item">
+        <span class="usa-breadcrumb__label"><a href="{{ url_for('.organizations') }}" class="usa-link navigation-organization-link">Organizations:</a></span>
+      </li>
+      <li class="usa-breadcrumb__list-item">
+        <span class="usa-breadcrumb__label">{{ current_org.name }}</span>
+      </li>
+      <a href="{{ url_for('main.choose_account') }}" class="usa-link navigation-service">Switch service</a>
+    </ol>
+  </nav>

app/templates/new/components/service_nav.html

@@ -0,0 +1,15 @@
+<div class="navigation-service margin-top-5 display-flex flex-align-end flex-justify border-bottom padding-bottom-1">
+  {% if current_service.organization_id %}
+    {% if current_user.platform_admin or
+      (current_user.belongs_to_organization(current_service.organization_id) and current_service.live) %}
+      <a href="{{ url_for('.organization_dashboard', org_id=current_service.organization_id) }}" class="usa-link navigation-organization-link">{{ current_service.organization_name }}</a>
+    {% endif %}
+  {% endif %}
+  <div class="font-body-2xl text-bold">
+    {{ current_service.name }}
+    {% if not current_service.active %}
+      <span class="navigation-service-name navigation-service-type--suspended">Suspended</span>
+    {% endif %}
+  </div>
+  <a href="{{ url_for('main.choose_account') }}" class="usa-link">Switch service</a>
+</div>

app/templates/new/layouts/withnav_template.html

@@ -1,41 +0,0 @@
-{% extends "base.html" %}
-
-{% block per_page_title %}
-  {% block service_page_title %}{% endblock %} – {{ current_service.name }}
-{% endblock %}
-
-{% block main %}
-  <div class="grid-container">
-    {% block serviceNavigation %}
-      {% include "service_navigation.html" %}
-    {% endblock %}
-    <!-- The withnav_template can be used to replace the settings_template. when it comes to setting_template and withnav_template, this service_navigation.html is only used in withnav_template and in settings_template, it was not used. That is one out of the two differences between settings template and withnav template. Child templates that extends settings_template, include the block serviceNavigation but leave it empty. Within the app, the only pages settings_template.html is used: manage-users.html, service-settings.html, and user-profile.html  -->
-    <div class="grid-row margin-top-5">
-      {% if help %}
-        <div class="grid-col-3">
-      {% else %}
-        <div class="grid-col-3">
-      {% endif %}
-        {% block sideNavigation %}
-          {% include "main_nav.html" %}
-          <!-- include settings_nav.html for child templates that used settings_template -->
-        {% endblock %}
-        </div>
-      {% if help %}
-        <div class="grid-col-8">
-      {% else %}
-        <div class="grid-col-9 padding-left-4">
-      {% endif %}
-        {% block beforeContent %}
-          {% block backLink %}{% endblock %}
-        {% endblock %}
-        <main id="main-content" role="main" class="usa-prose site-prose margin-bottom-10">
-        {% block content %}
-          {% include 'flash_messages.html' %}
-          {% block maincolumn_content %}{% endblock %}
-        {% endblock %}
-        </main>
-      </div>
-    </div>
-  </div>
-{% endblock %}

app/templates/new/templates_glossary.md

@@ -6,33 +6,31 @@ This document serves as a glossary for the templates directory structure of the
 ## Directory Structure
 
 - `/templates`
-  - `base.html`: The main base template from which all other templates inherit. This template is a combination of `main_template`, `admin_template`, `withoutnav_template`, `settings_templates`, and `content_template`.
+  - `base.html`: The main base template from which all other templates inherit. This template is a combination of `main_template`, `admin_template`, `withoutnav_template`, and `content_template`.
   - **/layouts**: Contains shared layouts used across the site. Simply put, it defines the overall structure or skeleton of the application (less frequently revised).
-    - `withnav_template.html`: A variation of the base layout that includes a sidebar.
-    - `org_template.html`: A variaton of the withnav_template
+    - `withnav_template.html`: A variation of org_template and settings_template that includes a sidebar.
   - **/components**: Houses reusable UI components that can be included in multiple templates and can be tailored with different content or links depending on the context.(more frequently revised or customized)
     - `head.html`: Template for the site's <head>, included in `base.html`.
     - `header.html`: Template for the site's header, included in `base.html`.
     - `footer.html`: Template for the site's footer, included in `base.html`.
+    - `settings_navigation.html`: The settings navigation used in `withnav_template.html` that previously extended `settings_template.html`.
+    - `org_nav.html`: The organization's navigation used solely in `org_template.html`.
+    - `main_nav.html`: The main navigation used in `withnav_template.html`
+    - `service_navigation.html`: The service navigation used in `withnav_template.html`. In withnav_template.html, the `serviceNavigation` block will be left empty in any child templates that previously extended `settings_template.html`.
   - **/views** (or **/pages**): Individual page templates that use the base layouts, components, and partials to present content.
 
 ### Best Practices
 
 - Use **inheritance** `{% extends %}` to build on base layouts.
-- Employ **components** `{% include %}` for reusable UI elements to keep the code DRY and facilitate easier updates.
-
-### Observation Notes
-- The macro-options.json files in the header and footer component act as structural guides. They aren't directly used as data passed to the usaFooter function/macro. Instead, these files outline the expected properties and provide a description of their purpose. The `usaFooter` macro component is currently only invoked in the `admin_template`, which will eventually serve as the `base.html` template. This will simplify the approach when we change the footer macros to componenets by eliminating the need to dynamically pass this data from the base.html template. This is also true for `usaHeader` macro component.
-
-
+- Employ **components** `{% include %}` for reusable UI elements.
 
 ### Old Layout Templates We Don't Need
-- withoutnav_template.html Delete
-- main_template.html Delete
+- withoutnav_template.html Delete, it is now apart of base.html
+- main_template.html Delete, it is now apart of base.html
 - settings_templates.html `withnav_template` can be used to replace `settings_template`.
-- settings_nav.html (move to /new/navigation directory)
-- main_nav.html (move to /new/navigation directory)
-- service_navigation.html (move to /new/navigation directory)
-- org_template, could be under it's own directory called /layout/organization
-- org_nav.html (move to /new/navigation directory)
-- content_template.html Delete
+- settings_nav.html (move to /components/ directory)
+- main_nav.html (move to /components/ directory)
+- service_navigation.html (move to /components/ directory)
+- org_template, Delete, it is now apart of withnav_template.html
+- org_nav.html (move to /components/ directory)
+- content_template.html Delete, it is now apart of base.html

app/templates/old/admin_template.html

@@ -291,10 +291,12 @@
 
 {% block bodyEnd %}
   {% block extra_javascripts %}
+
   {% endblock %}
   <!--[if gt IE 8]><!-->
   <script type="text/javascript" src="{{ asset_url('javascripts/all.js') }}"></script>
   <script type="text/javascript" src="{{ asset_url('js/uswds.min.js') }}"></script>
+
   <!--<![endif]-->
 
 {% endblock %}

app/templates/old/content_template.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/sub-navigation.html" import sub_navigation %}
 {% from "components/page-header.html" import page_header %}
 

app/templates/old/main_nav.html

@@ -1,3 +1,4 @@
+{# This template is an old version #}
 {% if help %}
   {% include 'partials/tour.html' %}
 {% else %}

app/templates/old/org_nav.html

@@ -1,3 +1,4 @@
+{# This template is an old version #}
 <nav class="navigation">
   <ul>
     <li><a class="usa-link{{ org_navigation.is_selected('dashboard') }}" href="{{ url_for('.organization_dashboard', org_id=current_org.id) }}">Usage</a></li>

app/templates/old/service_navigation.html

@@ -1,3 +1,4 @@
+{# This template is an old version #}
 {% macro navigation_service_name(service) %}
   <div class="font-body-2xl text-bold">
     {{ service.name }}

app/templates/old/settings_nav.html

@@ -0,0 +1,17 @@
+{# This template is an old version #}
+{% if help %}
+{% include 'partials/tour.html' %}
+{% else %}
+<nav class="nav">
+  <ul class="usa-sidenav">
+    {# {% if current_user.has_permissions() %} #}
+    <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('settings') }}"
+      href="{{ url_for('main.service_settings', service_id=current_service.id) }}">General</a></li>
+    <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('user-profile') }}"
+      href="{{ url_for('main.user_profile', service_id=current_service.id) }}">User profile</a></li>
+    <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('team-members') }}"
+        href="{{ url_for('main.manage_users', service_id=current_service.id) }}">Team members</a></li>
+    {# {% endif %} #}
+  </ul>
+</nav>
+{% endif %}

app/templates/old/settings_template.html

@@ -1,3 +1,4 @@
+{# This template is an old version #}
 {% extends "admin_template.html" %}
 
 {% block per_page_title %}

app/templates/old/withnav_template.html

@@ -0,0 +1,36 @@
+{# This template is an old version #}
+{% extends "admin_template.html" %}
+
+{% block per_page_title %}
+  {% block service_page_title %}{% endblock %} – {{ current_service.name }}
+{% endblock %}
+
+{% block main %}
+  <div class="grid-container">
+    {% include "service_navigation.html" %}
+    <div class="grid-row margin-top-5">
+      {% if help %}
+        <div class="tablet:grid-col-3">
+      {% else %}
+        <div class="tablet:grid-col-3">
+      {% endif %}
+          {% include "main_nav.html" %}
+        </div>
+      {% if help %}
+        <div class="grid-col-8">
+      {% else %}
+        <div class="tablet:grid-col-9 tablet:padding-left-4">
+      {% endif %}
+        {% block beforeContent %}
+          {% block backLink %}{% endblock %}
+        {% endblock %}
+        <main id="main-content" role="main" class="usa-prose site-prose margin-bottom-10">
+        {% block content %}
+          {% include 'flash_messages.html' %}
+          {% block maincolumn_content %}{% endblock %}
+        {% endblock %}
+        </main>
+      </div>
+    </div>
+  </div>
+{% endblock %}

app/templates/old/withoutnav_template.html

@@ -1,4 +1,4 @@
-{% extends "admin_template.html" %}
+{% extends "base.html" %}
 
 {% set mainClasses = "margin-top-5 padding-bottom-5" %}
 

app/templates/org_template.html

@@ -1,35 +0,0 @@
-{% extends "admin_template.html" %}
-
-{% block per_page_title %}
-  {% block org_page_title %}{% endblock %} – {{ current_org.name }}
-{% endblock %}
-
-{% block main %}
-  <div class="grid-container">
-    <div class="navigation-service usa-breadcrumb">
-      {% if current_user.platform_admin %}
-        <a href="{{ url_for('.organizations') }}" class="usa-link navigation-organization-link">Organizations</a>
-      {% endif %}
-      <div class="navigation-service">
-        {{ current_org.name }}
-      </div>
-      <a href="{{ url_for('main.choose_account') }}" class="usa-link navigation-service">Switch service</a>
-    </div>
-    <div class="grid-row">
-      <div class="grid-col-3">
-        {% include "org_nav.html" %}
-      </div>
-      <div class="grid-col-9">
-        {% block beforeContent %}
-          {% block backLink %}{% endblock %}
-        {% endblock %}
-        <main class="main" id="main-content" role="main" >
-          {% block content %}
-            {% include 'flash_messages.html' %}
-            {% block maincolumn_content %}{% endblock %}
-          {% endblock %}
-        </main>
-      </div>
-    </div>
-  </div>
-{% endblock %}

app/templates/views/accessibility_statement.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/service-link.html" import service_link %}
 {% from "components/content-metadata.html" import content_metadata %}
 
@@ -168,7 +168,7 @@
 
   <p>
     The following content on the <a class="usa-link" href="https://status.notifications.service.gov.uk/">Notify status page</a> is not compliant with the WCAG version 2.1 AA standard:
-  </p> 
+  </p>
 
   <ul class="usa-list">
     <li>an incorrect heading hierarchy which fails <a class="usa-link" href="https://www.w3.org/TR/WCAG21/#info-and-relationships">success criterion 1.3.1: info and relationships</a></li>

app/templates/views/add-service-local.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/add-service.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/cancelled-invitation.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% block per_page_title %}The invitation you were sent has been cancelled{% endblock %}
 {% block maincolumn_content %}
     <div class="grid-row">

app/templates/views/change-email-continue.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 
 {% block per_page_title %}
   Check your email

app/templates/views/check/preview.html

@@ -21,14 +21,14 @@
 
   {{ page_header('Preview') }}
   <div>
-    <p class="sms-message-scheduler">Scheduled: {{ scheduled_for |format_datetime_normal if scheduled_for else 'Now'}}</p>
+    <p class="sms-message-scheduler">Scheduled: {{ scheduled_for |format_datetime_scheduled_notification if scheduled_for else 'Now'}}</p>
     <p class="sms-message-file-name">File: {{original_file_name}}</p>
     <p class="sms-message-template">Template: {{template.name}}</p>
     <p class="sms-message-sender" >From: {{ template.sender }}</p>
   </div>
 
   <h2 id="{{ file_contents_header_id }}">Message</h2>
-  <div class="preview-message"> {{ simplifed_template|string }}</div>
+  <div class="preview-message"> {{ template|string }}</div>
   {% if not request.args.from_test %}
   <h2>Recipients list</h2>
   <div>

app/templates/views/choose-account.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/components/button/macro.njk" import usaButton %}
 
 {% macro service_list(

app/templates/views/dashboard/dashboard.html

@@ -22,6 +22,8 @@
       Messages sent
     </h2>
 
+    <!-- <button id="sevenDaysButton">7 Days</button>
+    <canvas id="myChart"></canvas> -->
     {{ ajax_block(partials, updates_url, 'inbox') }}
 
     {{ ajax_block(partials, updates_url, 'totals') }}
@@ -59,7 +61,7 @@
                 {% set notification = job.notifications[0] %}
                 <tr class="table-row" id="{{ job.job_id }}">
                   <td class="table-field file-name">
-                    {{ notification.job.original_file_name if notification.job.original_file_name else 'Manually entered number'}}
+                    {{ notification.job.original_file_name[:12] if notification.job.original_file_name else 'Manually entered number'}}
                     <br>
                     <a class="usa-link file-list-filename" href="{{ job.view_job_link }}">View Batch</a>
                   </td>

app/templates/views/documentation.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 
 {% block per_page_title %}
   Documentation

app/templates/views/email-link-interstitial.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 

app/templates/views/email-link-invalid.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-footer.html" import page_footer %}
 
 {% block per_page_title %}

app/templates/views/email-not-received.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/components/button/macro.njk" import usaButton %}
 
 {% block per_page_title %}

app/templates/views/features.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/table.html" import mapping_table, row, text_field, edit_field, field with context %}
 {% from "components/sub-navigation.html" import sub_navigation %}
 {% from "components/service-link.html" import service_link %}

app/templates/views/features/text-messages.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/table.html" import mapping_table, row, text_field, edit_field, field with context %}
 {% from "components/service-link.html" import service_link %}
 

app/templates/views/forgot-password.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}
 

app/templates/views/get-started.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/table.html" import mapping_table, row, text_field, edit_field, field with context %}
 {% from "components/sub-navigation.html" import sub_navigation %}
 {% from "components/components/details/macro.njk" import usaDetails %}

app/templates/views/guidance/create-and-send-messages.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/service-link.html" import service_link %}
 
 {% block per_page_title %}

app/templates/views/guidance/edit-and-format-messages.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/service-link.html" import service_link %}
 
 {% block per_page_title %}

app/templates/views/guidance/index.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/table.html" import mapping_table, row, text_field, edit_field, field with context %}
 {% from "components/sub-navigation.html" import sub_navigation %}
 {% from "components/service-link.html" import service_link %}

app/templates/views/guidance/send-files-by-email.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 
 {% block per_page_title %}
   Send files by email

app/templates/views/integration-testing.html

@@ -1,5 +1,5 @@
 {% from "components/table.html" import mapping_table, row, text_field, edit_field, field with context %}
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 
 {% block per_page_title %}
   Integration testing

app/templates/views/manage-users.html

@@ -1,4 +1,4 @@
-{% extends "settings_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/tick-cross.html" import tick_cross %}
 {% from "components/live-search.html" import live_search %}
 {% from "components/components/button/macro.njk" import usaButton %}
@@ -7,6 +7,12 @@
   Team members
 {% endblock %}
 
+{% block serviceNavigation %}{% endblock %}
+
+{% block sideNavigation %}
+  {% include "/new/components/settings_nav.html" %}
+{% endblock %}
+
 {% block maincolumn_content %}
 
   <div class="button-flex-header">
@@ -31,67 +37,70 @@
 
   <div class="user-list">
     {% for user in users %}
-      <div class="user-list-item">
-        <h2 class="user-list-item-heading font-body-lg margin-top-0" title="{{ user.email_address }}">
-          {%- if user.name -%}
-          <span class="heading-small live-search-relevant">{{ user.name }}</span>
-          {%- endif -%}
-          {%- if user.status == 'pending' -%}
-            <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(invited)</span>
-          {%- elif user.status == 'cancelled' -%}
-            <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(cancelled invite)</span>
-          {%- elif user.status == 'expired' -%}
-            <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(expired invite)</span>
-          {%- elif user.id == current_user.id -%}
-            <span class="live-search-relevant"></span><span class="hint">(you)</span>
-          {% else %}
-            <span class="live-search-relevant">{{ user.email_address }}</span>
-          {% endif %}
-        </h2>
-        <h3 class="margin-bottom-0">Permissions</h3>
-        <ul class="tick-cross-list-permissions">
-          {% for permission, label in permissions %}
-            {{ tick_cross(
-              user.has_permission_for_service(current_service.id, permission),
-              label
-            ) }}
-          {% endfor %}
-        </ul>
-        {# only show if the service has folders #}
-        {% if current_service.all_template_folders %}
-          <p class="usa-body tick-cross-list-hint">
-            {% set folder_count = user.template_folders_for_service(current_service) | length %}
-            {% if folder_count == 0 %}
-              Cannot see any folders
-            {% elif folder_count != current_service.all_template_folders | length %}
-              Can see {{ folder_count }} folder{% if folder_count > 1 %}s{% endif %}
+      {% if user.status != 'cancelled' %}
+        <div class="user-list-item">
+          <h2 class="user-list-item-heading font-body-lg margin-top-0" title="{{ user.email_address }}">
+            {%- if user.name -%}
+            <span class="heading-small live-search-relevant">{{ user.name }}</span>
+            {%- endif -%}
+            {%- if user.status == 'pending' -%}
+              <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(invited)</span>
+            {%- elif user.status == 'cancelled' -%}
+              <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(cancelled invite)</span>
+            {%- elif user.status == 'expired' -%}
+              <span class="live-search-relevant">{{ user.email_address }}</span><span class="hint">(expired invite)</span>
+            {%- elif user.id == current_user.id -%}
+              <span class="live-search-relevant"></span><span class="hint">(you)</span>
             {% else %}
-              Can see all folders
-            {% endif%}
-          </p>
-        {% endif %}
-        {% if current_service.has_permission('email_auth') %}
-          <p class="usa-body tick-cross-list-hint">
-            Signs in with
-            {{ user.auth_type | format_auth_type(with_indefinite_article=True) }}
-          </p>
-        {% endif %}
-        {% if current_service.has_permission('email_auth') %}
-          <p class="usa-body tick-cross-list-hint">
-            Signs in with
-            {{ user.auth_type | format_auth_type(with_indefinite_article=True) }}
-          </p>
-          {% endif %}
-            {% if current_user.has_permissions('manage_service') %}
-              {% if user.status == 'pending' %}
-                <a class="user-list-edit-link usa-link" href="{{ url_for('.cancel_invited_user', service_id=current_service.id, invited_user_id=user.id)}}">Cancel invitation<span class="usa-sr-only"> for {{ user.email_address }}</span></a>
-              {% elif user.status == 'expired' %}
-                <a class="user-list-edit-link usa-link" href="{{ url_for('.resend_invite', service_id=current_service.id, invited_user_id=user.id)}}">Resend invite<span class="usa-sr-only"> for {{ user.email_address }}</span></a>
-              {% elif user.is_editable_by(current_user) %}
-                <a class="user-list-edit-link usa-link" href="{{ url_for('.edit_user_permissions', service_id=current_service.id, user_id=user.id)}}">Change details<span class="usa-sr-only"> for {{ user.name }} {{ user.email_address }}</span></a>
+              <span class="live-search-relevant">{{ user.email_address }}</span>
             {% endif %}
+          </h2>
+          <h3 class="margin-bottom-0">Permissions</h3>
+          <ul class="tick-cross-list-permissions">
+            {% for permission, label in permissions %}
+              {{ tick_cross(
+                user.has_permission_for_service(current_service.id, permission),
+                label
+              ) }}
+            {% endfor %}
+          </ul>
+          {# only show if the service has folders #}
+          {% if current_service.all_template_folders %}
+            <p class="usa-body tick-cross-list-hint">
+              {% set folder_count = user.template_folders_for_service(current_service) | length %}
+              {% if folder_count == 0 %}
+                Cannot see any folders
+              {% elif folder_count != current_service.all_template_folders | length %}
+                Can see {{ folder_count }} folder{% if folder_count > 1 %}s{% endif %}
+              {% else %}
+                Can see all folders
+              {% endif%}
+            </p>
           {% endif %}
-      </div>
+          {% if current_service.has_permission('email_auth') %}
+            <p class="usa-body tick-cross-list-hint">
+              Signs in with
+              {{ user.auth_type | format_auth_type(with_indefinite_article=True) }}
+            </p>
+          {% endif %}
+          {% if current_service.has_permission('email_auth') %}
+            <p class="usa-body tick-cross-list-hint">
+              Signs in with
+              {{ user.auth_type | format_auth_type(with_indefinite_article=True) }}
+            </p>
+            {% endif %}
+              {% if current_user.has_permissions('manage_service') %}
+                {% if user.status == 'pending' or user.status == 'expired' %}
+                  <a class="user-list-edit-link usa-link" href="{{ url_for('.cancel_invited_user', service_id=current_service.id, invited_user_id=user.id)}}">Cancel invitation<span class="usa-sr-only"> for {{ user.email_address }}</span></a>
+                {% endif %}
+                {% if user.status == 'expired' %}
+                  <a class="user-list-edit-link usa-link" href="{{ url_for('.resend_invite', service_id=current_service.id, invited_user_id=user.id)}}">Resend invite<span class="usa-sr-only"> for {{ user.email_address }}</span></a>
+                {% elif user.is_editable_by(current_user) %}
+                  <a class="user-list-edit-link usa-link" href="{{ url_for('.edit_user_permissions', service_id=current_service.id, user_id=user.id)}}">Change details<span class="usa-sr-only"> for {{ user.name }} {{ user.email_address }}</span></a>
+                {% endif %}
+            {% endif %}
+        </div>
+      {% endif %}
     {% endfor %}
   </div>
 

app/templates/views/message-status.html

@@ -1,4 +1,4 @@
-{% extends "content_template.html" %}
+{% extends "base.html" %}
 {% from "components/table.html" import mapping_table, row, text_field %}
 
 {% block per_page_title %}
@@ -9,9 +9,13 @@
 
   <h1 class="font-body-2xl margin-bottom-3">Delivery status</h1>
 
-  <p>Notify’s real-time dashboard lets you check the status of any message.</p>
-  <p>For <a class="usa-link" href="{{ url_for('main.security') }}">security</a>, this information is only available for seven days after a message has been sent. You can download a report, including a list of sent messages, for your own records.</p>
-  <p>This page describes the statuses you'll see when you're signed in to Notify.</p>
+  <p>Notify starts sending your text messages immediately. Each message is sent to its cell phone carrier, which attempts
+  delivery to the recipient. This process is often almost instantaneous, but can sometimes take a while if a phone is
+  unavailable. The carrier will continue to try to deliver the message for up to 72 hours.</p>
+  <p>The Notify dashboard provides a high-level view of the number of messages Sent, Pending, Delivered, and Failed. The
+  dashboard data starts to update about five minutes after a message is sent and will continue to update for four hours.</p>
+  <p>Delivery statuses reflect delivery to a specific <em>cell phone number</em>, not to a <em>person</em>. No status can confirm that a
+  specific person actually received or read the message.</p>
 
   <!-- <p>If you’re using the Notify API, read our <a class="usa-link" href="{{ url_for('.documentation') }}">documentation</a> for a list of API statuses.<p>
 
@@ -45,7 +49,7 @@
   <p>Notify cannot tell you if your users open an email or click on the links in an email. We do not track open rates and click-throughs because there are privacy issues. Tracking emails without asking permission from users could breach General Data Protection Regulations (GDPR).</p> -->
 
   <h2 id="text-message-statuses" class="heading-medium">Text messages</h2>
-  <div class="bottom-gutter-3-2">
+  <div class="margin-bottom-8">
     {% call mapping_table(
       caption='Message statuses – text messages',
       field_headings=['Status', 'Description'],
@@ -53,12 +57,14 @@
       caption_visible=False
     ) %}
       {% for message_status, description in [
-        ('Total', 'The total number of messages that have been sent during the last seven days.'),
-        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72 hours. Notify is waiting for delivery information.'),
+        ('Scheduled', 'The total number of messages that have been scheduled to be sent at some future time.'),
+        ('Canceled', 'Messages that were created and scheduled, but canceled prior to Sending.'),
+        ('Total', 'The total number of messages that have been sent during the specified time.'),
+        ('Pending', 'Notify has sent the message to the provider. The provider will try to deliver the message to the recipient for up to 72
+        hours. “Pending” indicates that Notify is waiting for delivery information.'),
         ('Delivered', 'The message was successfully delivered. Notify cannot tell you if a user has opened or read a message.'),
-        ('Failed', 'The provider could not deliver the message. This can happen if the phone number was wrong or if the network operator
-        rejects the message. If you’re sure that these phone numbers are correct, you should <a class="usa-link" href="/support">contact us</a>. If not, you should remove them from your database. You’ll still be charged for text messages that
-        cannot be delivered.' | safe),
+        ('Failed', 'The message could not be delivered.'),
+        ('Process error / Delivery not attempted', 'If you receive a large number of process errors, please contact the Notify team.'),
         ] %}
         {% call row() %}
           {{ text_field(message_status) }}
@@ -67,5 +73,21 @@
       {% endfor %}
     {% endcall %}
   </div>
+  <h2 class="heading-medium">About carrier statuses</h2>
+  <p>Sometimes Notify receives more detailed information from the carriers on the status of messages, and these can be found
+  in the downloadable reports. Not all carriers provide the same level of detail regarding delivery and some delivery
+  statutes have a slight variation in word choice. Notify includes this information in the reports to provide you as much
+  detail as possible. Remember, for <a class="usa-link" href="/features/security">security</a> purposes, detailed information is only available for seven days after a
+  message has been sent.</p>
+
+  <h2 class="heading-medium">Opting out</h2>
+  <p>A text recipient can opt out of receiving text messages from your phone number at any time by responding “STOP” or
+  “QUIT” or <a class="usa-link" href="https://docs.aws.amazon.com/pinpoint/latest/userguide/channels-sms-limitations-opt-out.html#settings-sms-self-opt-out-terms">one of several other keywords</a>. If they opt out,</p>
+  <ul>
+    <li>They may receive a response from their carrier stating that they will no longer receive text messages from that number.</li>
+    <li>Any subsequent messages sent to that number will not be delivered (unless the recipient opts back in by texting “START”
+    or “OPT-IN”).</li>
+  </ul>
+  <p>Notify.gov does not yet have a way to pull opt-out status and make it available in the UI for agencies to download.</p>
 
 {% endblock %}

app/templates/views/new-password.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}
 

app/templates/views/notifications.html

@@ -64,10 +64,22 @@
 
   {% if current_user.has_permissions('view_activity') %}
     <p class="font-body-sm">
-      <a href="{{ download_link }}" download="download" class="usa-link">Download this report (<abbr title="Comma separated values">CSV</abbr>)</a>
+      <a href="{{ download_link_seven_day }}" download="download" class="usa-link">Download all data last 7 days (<abbr title="Comma separated values">CSV</abbr>)</a>
       &emsp;
       Data available for {{ partials.service_data_retention_days }} days
     </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_five_day }}" download="download" class="usa-link">Download all data last 5 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_three_day }}" download="download" class="usa-link">Download all data last 3 days (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
+    <p class="font-body-sm">
+      <a href="{{ download_link_today }}" download="download" class="usa-link">Download all data today (<abbr title="Comma separated values">CSV</abbr>)</a>
+      &emsp;
+    </p>
   {% endif %}
 
   {{ ajax_block(

app/templates/views/notifications/preview.html

@@ -5,11 +5,21 @@
 {% from "components/components/button/macro.njk" import usaButton %}
 
 {% block service_page_title %}
-  {{ "Error" if error else "Preview" }}
+  {% if error %}
+    {{ "Error" }}
+  {% elif help %}
+    {{ "Example text message" }}
+  {% else %}
+    {{ "Preview" }}
+  {% endif %}
 {% endblock %}
 
 {% block backLink %}
-  {{ usaBackLink({ "href": back_link_from_preview }) }}
+  {% if help %}
+    {{ usaBackLink({ "href": back_link }) }}
+  {% else %}
+    {{ usaBackLink({ "href": back_link_from_preview }) }}
+  {% endif %}
 {% endblock %}
 
 {% block maincolumn_content %}
@@ -40,17 +50,24 @@
       {% endcall %}
     </div>
   {% else %}
-    {{ page_header('Preview') }}
+    {{ page_header("Example text message" if help else "Preview") }}
   {% endif %}
-  <div>
-    <p class="sms-message-scheduler">Scheduled: {{ scheduled_for |format_datetime_normal if scheduled_for else 'Now'}}</p>
-    <p class="sms-message-template">Template: {{template.name}}</p>
-    <p class="sms-message-sender" >From: {{ template.sender }}</p>
-    <p class="sms-message-sender" >To: {{ recipient }}</p>
-  </div>
 
-  <h2 id="{{ file_contents_header_id }}">Message</h2>
-  <div class="preview-message"> {{ simplifed_template|string }}</div>
+  {% if not help %}
+    <div>
+      <p class="sms-message-scheduler">Scheduled: {{ scheduled_for |format_datetime_scheduled_notification if scheduled_for else 'Now'}}</p>
+      <p class="sms-message-template">Template: {{template.name}}</p>
+      <p class="sms-message-sender" >From: {{ template.sender }}</p>
+      <p class="sms-message-sender" >To: {{ recipient }}</p>
+    </div>
+  {% endif %}
+
+  {% if not help %}
+    <h2 class="message-header">Message</h2>
+  {% endif %}
+
+
+  <div class="preview-message"> {{ template|string }}</div>
 
   <div class="js-stick-at-bottom-when-scrolling">
     <form method="post" enctype="multipart/form-data" action="{{url_for(

app/templates/views/organizations/add-organization.html

@@ -1,4 +1,4 @@
-{% extends "withoutnav_template.html" %}
+{% extends "base.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/organizations/organization/billing.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-header.html" import page_header %}
 
 {% block org_page_title %}

app/templates/views/organizations/organization/index.html

@@ -2,7 +2,7 @@
 {% from "components/big-number.html" import big_number %}
 {% from "components/live-search.html" import live_search %}
 {% from "components/pill.html" import pill %}
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 
 {% block org_page_title %}
   Usage

app/templates/views/organizations/organization/settings/edit-domains.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/list-entry.html" import list_entry %}

app/templates/views/organizations/organization/settings/edit-name.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/organizations/organization/settings/edit-organization-billing-details.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/organizations/organization/settings/edit-organization-notes.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/form.html" import form_wrapper %}

app/templates/views/organizations/organization/settings/edit-type.html

@@ -1,4 +1,4 @@
-{% extends "org_template.html" %}
+{% extends "withnav_template.html" %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/page-header.html" import page_header %}
 {% from "components/form.html" import form_wrapper %}