diff --git a/.github/actions/deploy-proxy/action.yml b/.github/actions/deploy-proxy/action.yml index 13bdc494f..339d1fc78 100644 --- a/.github/actions/deploy-proxy/action.yml +++ b/.github/actions/deploy-proxy/action.yml @@ -16,6 +16,19 @@ inputs: runs: using: composite steps: + - name: Install cf-cli + shell: bash + run: | + curl -A "cg-deploy-action" -v -L -o cf-cli_amd64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github' + sudo dpkg -i cf-cli_amd64.deb + - name: Login to cf-cli + shell: bash + run: | + cf api api.fr.cloud.gov + cf auth + - name: Target org and space + shell: bash + run: cf target -o ${{ inputs.cf_org }} -s ${{ inputs.cf_space }} - name: Set restricted space egress shell: bash run: ./terraform/set_space_egress.sh -t -s ${{ inputs.cf_space }} diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 89adc1f29..31b9cec6b 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -47,7 +47,7 @@ jobs: run: poetry export --without-hashes --format=requirements.txt > requirements.txt - name: Deploy to cloud.gov - uses: 18f/cg-deploy-action@main + uses: cloud-gov/cg-cli-tools@main env: DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} SECRET_KEY: ${{ secrets.SECRET_KEY }} @@ -63,12 +63,14 @@ jobs: LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-demo.app.cloud.gov/sign-out" LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATE" + LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs" with: cf_username: ${{ secrets.CLOUDGOV_USERNAME }} cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} cf_org: gsa-tts-benefits-studio cf_space: notify-demo push_arguments: >- + push -f manifest.yml --vars-file deploy-config/demo.yml --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" @@ -85,6 +87,8 @@ jobs: --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" + --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL" + --strategy rolling - name: Check for changes to egress config id: changed-egress-config diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 262079be8..dd71b1e11 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -47,7 +47,7 @@ jobs: run: poetry export --without-hashes --format=requirements.txt > requirements.txt - name: Deploy to cloud.gov - uses: 18f/cg-deploy-action@main + uses: cloud-gov/cg-cli-tools@main env: DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} SECRET_KEY: ${{ secrets.SECRET_KEY }} @@ -63,12 +63,14 @@ jobs: LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://beta.notify.gov/sign-out" LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/sign-in&response_type=code&scope=openid+email&state=STATE" + LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs" with: cf_username: ${{ secrets.CLOUDGOV_USERNAME }} cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} cf_org: gsa-tts-benefits-studio cf_space: notify-production push_arguments: >- + push -f manifest.yml --vars-file deploy-config/production.yml --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" @@ -85,6 +87,8 @@ jobs: --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" + --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL" + --strategy rolling - name: Check for changes to egress config id: changed-egress-config diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8cf33babc..4da91e0f3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,11 +53,12 @@ jobs: - name: Deploy to cloud.gov - uses: 18f/cg-deploy-action@main + uses: cloud-gov/cg-cli-tools@main env: DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} SECRET_KEY: ${{ secrets.SECRET_KEY }} ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + ADMIN_CLIENT_USERNAME: "notify-admin" NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }} COMMIT_HASH: ${{ github.sha }} @@ -69,21 +70,22 @@ jobs: LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?" LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out" LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=STATEE" + LOGIN_DOT_GOV_CERTS_URL: "https://secure.login.gov/api/openid_connect/certs" with: cf_username: ${{ secrets.CLOUDGOV_USERNAME }} cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} cf_org: gsa-tts-benefits-studio cf_space: notify-staging - push_arguments: >- + cf_command: >- + push -f manifest.yml --vars-file deploy-config/staging.yml --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" - --var ADMIN_CLIENT_USERNAME="notify-admin" --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" + --var ADMIN_CLIENT_USERNAME="$ADMIN_CLIENT_USERNAME" --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY" --var NR_BROWSER_KEY="$NR_BROWSER_KEY" --var COMMIT_HASH="$COMMIT_HASH" - --var LOGIN_PEM="$LOGIN_PEM" --var LOGIN_DOT_GOV_CLIENT_ID="$LOGIN_DOT_GOV_CLIENT_ID" --var LOGIN_DOT_GOV_USER_INFO_URL="$LOGIN_DOT_GOV_USER_INFO_URL" --var LOGIN_DOT_GOV_ACCESS_TOKEN_URL="$LOGIN_DOT_GOV_ACCESS_TOKEN_URL" @@ -91,6 +93,9 @@ jobs: --var LOGIN_DOT_GOV_BASE_LOGOUT_URL="$LOGIN_DOT_GOV_BASE_LOGOUT_URL" --var LOGIN_DOT_GOV_SIGNOUT_REDIRECT="$LOGIN_DOT_GOV_SIGNOUT_REDIRECT" --var LOGIN_DOT_GOV_INITIAL_SIGNIN_URL="$LOGIN_DOT_GOV_INITIAL_SIGNIN_URL" + --var LOGIN_DOT_GOV_CERTS_URL="$LOGIN_DOT_GOV_CERTS_URL" + --var LOGIN_PEM="$LOGIN_PEM" + --strategy rolling - name: Check for changes to egress config @@ -102,7 +107,7 @@ jobs: .github/actions/deploy-proxy/action.yml .github/workflows/deploy.yml - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' + #if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy with: cf_space: notify-staging diff --git a/.gitignore b/.gitignore index d0a03e6f9..aaa99d325 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,8 @@ playwright/ # Nodenv .node-version + +# BackstopJS +backstop_data/bitmaps_test/ +backstop_data/html_report/ +backstop_data/engine_scripts/playwright/ diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000..544138be4 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} diff --git a/app/assets/error_pages/5xx.html b/app/assets/error_pages/5xx.html index b84e0f729..7b2cca9c1 100644 --- a/app/assets/error_pages/5xx.html +++ b/app/assets/error_pages/5xx.html @@ -68,11 +68,8 @@

Try again later.

-

- You can check our system status page to see if there are any known - issues.
To report a problem, email gov-uk-notify-support@digital.cabinet-office.gov.uk +

To report a problem you can email us at notify-support@gsa.gov.

+

You can expect a response within one business day.

diff --git a/app/content/get-started.md b/app/content/get-started.md index 47757b2c1..7d18aacbd 100644 --- a/app/content/get-started.md +++ b/app/content/get-started.md @@ -14,7 +14,7 @@ Learn how to [personalize messages](/using-notify/guidance) to increase response Learn about message _parts_ and [how limits are calculated](/using-notify/pricing). 5. ## Start sending messages -To remove the restrictions of Trial Mode and begin sending messages to clients complete the [Live Campaign Form](https://airtable.com/appe4n7jYOALPLcyU/shrIPWnLTw9U1fclL). +To remove the restrictions of Trial Mode and begin sending messages to clients complete the Go-Live Form. We’ll respond within one business day. ### Questions? diff --git a/app/main/views/index.py b/app/main/views/index.py index e59170aa5..284c7b768 100644 --- a/app/main/views/index.py +++ b/app/main/views/index.py @@ -1,6 +1,15 @@ import os +import secrets -from flask import abort, current_app, redirect, render_template, request, url_for +from flask import ( + abort, + current_app, + redirect, + render_template, + request, + session, + url_for, +) from flask_login import current_user from app import status_api_client @@ -27,8 +36,12 @@ def index(): ) url = os.getenv("LOGIN_DOT_GOV_INITIAL_SIGNIN_URL") # handle unit tests + + nonce = secrets.token_urlsafe() + session["nonce"] = nonce + if url is not None: - url = url.replace("NONCE", token) + url = url.replace("NONCE", nonce) url = url.replace("STATE", token) return render_template( "views/signedout.html", diff --git a/app/main/views/send.py b/app/main/views/send.py index 7899af5a8..2582cfeff 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -1009,9 +1009,16 @@ def _send_notification(service_id, template_id): keys = [] values = [] + # Guarantee that the real phone number comes last, because some + # users will have placeholders like "add your second phone number" + # or something like as custom placeholders. for k, v in session["placeholders"].items(): - keys.append(k) - values.append(v) + if k != "phone number": + keys.append(k) + values.append(v) + if "phone number" in session["placeholders"].keys(): + keys.append("phone number") + values.append(session["placeholders"]["phone number"]) data = ",".join(keys) vals = ",".join(values) @@ -1027,7 +1034,7 @@ def _send_notification(service_id, template_id): # on the API side to find out what happens to the message. current_app.logger.info( hilite( - f"One-off file: {filename} job_id: {upload_id} s3 location: service-{service_id}-notify/{upload_id}.csv" + f"One-off file: {filename} job_id: {upload_id} s3 location: {service_id}-service-notify/{upload_id}.csv" ) ) diff --git a/app/main/views/sign_in.py b/app/main/views/sign_in.py index 840e9fdc4..6f8d83609 100644 --- a/app/main/views/sign_in.py +++ b/app/main/views/sign_in.py @@ -1,4 +1,6 @@ +import json import os +import secrets import time import uuid @@ -12,6 +14,7 @@ from flask import ( redirect, render_template, request, + session, url_for, ) from flask_login import current_user @@ -28,7 +31,7 @@ from app.utils.user import is_gov_user from notifications_utils.url_safe_token import generate_token -def _reformat_keystring(orig): +def _reformat_keystring(orig): # pragma: no cover arr = orig.split("-----") begin = arr[1] end = arr[3] @@ -37,9 +40,10 @@ def _reformat_keystring(orig): return new_keystring -def _get_access_token(code, state): +def _get_access_token(code, state): # pragma: no cover client_id = os.getenv("LOGIN_DOT_GOV_CLIENT_ID") access_token_url = os.getenv("LOGIN_DOT_GOV_ACCESS_TOKEN_URL") + certs_url = os.getenv("LOGIN_DOT_GOV_CERTS_URL") keystring = os.getenv("LOGIN_PEM") if " " in keystring: keystring = _reformat_keystring(keystring) @@ -60,17 +64,47 @@ def _get_access_token(code, state): url = f"{base_url}{cli_assert}&{cli_assert_type}&{code_param}&grant_type=authorization_code" headers = {"Authorization": "Bearer %s" % token} response = requests.post(url, headers=headers) - if response.json().get("access_token") is None: - # Capture the response json here so it hopefully shows up in error reports - current_app.logger.error( + + response_json = response.json() + try: + encoded_id_token = response_json["id_token"] + except KeyError as e: + current_app.logger.exception(f"Error when getting id token {response_json}") + raise KeyError(f"'access_token' {response.json()}") from e + + # Getting Login.gov signing keys for unpacking the id_token correctly. + jwks = requests.get(certs_url).json() + public_keys = { + jwk["kid"]: { + "key": jwt.algorithms.RSAAlgorithm.from_jwk(json.dumps(jwk)), + "algo": jwk["alg"], + } + for jwk in jwks["keys"] + } + kid = jwt.get_unverified_header(encoded_id_token)["kid"] + pub_key = public_keys[kid]["key"] + algo = public_keys[kid]["algo"] + id_token = jwt.decode( + encoded_id_token, pub_key, audience=client_id, algorithms=[algo] + ) + + nonce = id_token["nonce"] + saved_nonce = session.pop("nonce") + if nonce != saved_nonce: + current_app.logger.error(f"Nonce Error: {nonce} != {saved_nonce}") + abort(403) + + try: + access_token = response_json["access_token"] + except KeyError as e: + current_app.logger.exception( f"Error when getting access token {response.json()} #notify-admin-1505" ) - raise KeyError(f"'access_token' {response.json()}") - access_token = response.json()["access_token"] + raise KeyError(f"'access_token' {response.json()}") from e return access_token -def _get_user_email_and_uuid(access_token): +def _get_user_email_and_uuid(access_token): # pragma: no cover headers = {"Authorization": "Bearer %s" % access_token} user_info_url = os.getenv("LOGIN_DOT_GOV_USER_INFO_URL") user_attributes = requests.get( @@ -82,7 +116,7 @@ def _get_user_email_and_uuid(access_token): return user_email, user_uuid -def _do_login_dot_gov(): +def _do_login_dot_gov(): # $ pragma: no cover # start login.gov code = request.args.get("code") state = request.args.get("state") @@ -124,12 +158,13 @@ def _do_login_dot_gov(): except BaseException as be: # noqa B036 current_app.logger.error(f"Error signing in: {be} #notify-admin-1505 ") error(401) + return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url)) # end login.gov -def verify_email(user, redirect_url): +def verify_email(user, redirect_url): # pragma: no cover user_api_client.send_verify_code(user["id"], "email", None, redirect_url) title = "Email resent" if request.args.get("email_resent") else "Check your email" redirect_url = request.args.get("next") @@ -138,7 +173,7 @@ def verify_email(user, redirect_url): ) -def _handle_e2e_tests(redirect_url): +def _handle_e2e_tests(redirect_url): # pragma: no cover try: current_app.logger.warning("E2E TESTS ARE ENABLED.") current_app.logger.warning( @@ -146,26 +181,32 @@ def _handle_e2e_tests(redirect_url): ) user = user_api_client.get_user_by_email(os.getenv("NOTIFY_E2E_TEST_EMAIL")) activate_user(user["id"]) + + # Check if the redirect URL is present and safe before proceeding further + if redirect_url and is_safe_redirect_url(redirect_url): + return redirect(redirect_url) + return redirect( url_for( "main.show_accounts_or_dashboard", next="EMAIL_IS_OK", ) ) + except Exception as e: stre = str(e) stre = stre.replace(" ", "_") - # Trying to get a message back to playwright somehow since we can't see the admin logs + # Trying to get a message back to playwright somehow since we can't raise an error return redirect(url_for(f"https://{stre}")) @main.route("/sign-in", methods=(["GET", "POST"])) @hide_from_search_engines -def sign_in(): +def sign_in(): # pragma: no cover redirect_url = request.args.get("next") if os.getenv("NOTIFY_E2E_TEST_EMAIL"): - return _handle_e2e_tests(None) + return _handle_e2e_tests(redirect_url) # If we have to revalidated the email, send the message # via email and redirect to the "verify your email page" @@ -189,10 +230,15 @@ def sign_in(): current_app.config["DANGEROUS_SALT"], ) url = os.getenv("LOGIN_DOT_GOV_INITIAL_SIGNIN_URL") + + nonce = secrets.token_urlsafe() + session["nonce"] = nonce + # handle unit tests if url is not None: - url = url.replace("NONCE", token) + url = url.replace("NONCE", nonce) url = url.replace("STATE", token) + return render_template( "views/signin.html", again=bool(redirect_url), @@ -201,5 +247,5 @@ def sign_in(): @login_manager.unauthorized_handler -def sign_in_again(): +def sign_in_again(): # pragma: no cover return redirect(url_for("main.sign_in", next=request.path)) diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index 752f054a4..09454ade0 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -10,13 +10,13 @@ from app.s3_client import ( ) from notifications_utils.s3 import s3upload as utils_s3upload -FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv" +NEW_FILE_LOCATION_STRUCTURE = "{}-service-notify/{}.csv" def get_csv_location(service_id, upload_id): return ( current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], - FILE_LOCATION_STRUCTURE.format(service_id, upload_id), + NEW_FILE_LOCATION_STRUCTURE.format(service_id, upload_id), current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"], current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"], current_app.config["CSV_UPLOAD_BUCKET"]["region"], diff --git a/app/templates/components/components/back-link/template.njk b/app/templates/components/components/back-link/template.njk index 25e611b06..aba4e4d95 100644 --- a/app/templates/components/components/back-link/template.njk +++ b/app/templates/components/components/back-link/template.njk @@ -1,2 +1,4 @@ -{{ (params.html | safe if params.html else (params.text if params.text else 'Back')) }} + diff --git a/app/templates/components/components/skip-link/template.njk b/app/templates/components/components/skip-link/template.njk index f18f881e8..a875c5d66 100644 --- a/app/templates/components/components/skip-link/template.njk +++ b/app/templates/components/components/skip-link/template.njk @@ -1,7 +1,3 @@ {{- params.html | safe if params.html else params.text -}} - - - Skip to main content - \ No newline at end of file diff --git a/app/templates/components/main_nav.html b/app/templates/components/main_nav.html index 597ccf45b..118442095 100644 --- a/app/templates/components/main_nav.html +++ b/app/templates/components/main_nav.html @@ -1,7 +1,7 @@ {% if help %} {% include 'partials/tour.html' %} {% else %} -