Revert "Production deploy 5/10/2024"

Production deploy 5/10/2024

README.md

@@ -141,11 +141,10 @@ Terraform installations. This is great, but you still need to install Terraform
 itself, which can be done with this command:
 
 ```sh
-tfenv install latest:^1.4.0
+tfenv install "latest:^1.7"
+tfenv use 1.7.x # x = the patch version installed
 ```
 
-_NOTE: This project currently uses the latest `1.4.x release of Terraform._
-
 #### Python Installation
 
 Now we're going to install a tool to help us manage Python versions and

app/organization/invite_rest.py

@@ -89,14 +89,6 @@ def invite_user_to_org(organization_id):
         ex=1800,
     )
 
-    # This is for the login.gov path, note 24 hour expiry to match
-    # The expiration of invitations.
-    redis_key = f"organization-invite-{invited_org_user.email_address}"
-    redis_store.set(
-        redis_key,
-        organization_id,
-        ex=3600 * 24,
-    )
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
     return jsonify(data=invited_org_user.serialize()), 201

app/service/rest.py

@@ -102,7 +102,7 @@ from app.service.service_senders_schema import (
 )
 from app.service.utils import get_guest_list_objects
 from app.user.users_schema import post_set_permissions_schema
-from app.utils import get_prev_next_pagination_links
+from app.utils import get_prev_next_pagination_links, hilite
 
 service_blueprint = Blueprint("service", __name__)
 
@@ -314,7 +314,9 @@ def get_users_for_service(service_id):
 def add_user_to_service(service_id, user_id):
     service = dao_fetch_service_by_id(service_id)
     user = get_user_by_id(user_id=user_id)
-
+    # TODO REMOVE DEBUG
+    print(hilite(f"GOING TO ADD {user.name} to service {service.name}"))
+    # END DEBUG
     if user in service.users:
         error = "User id: {} already part of service id: {}".format(user_id, service_id)
         raise InvalidRequest(error, status_code=400)
@@ -329,6 +331,10 @@ def add_user_to_service(service_id, user_id):
     folder_permissions = data.get("folder_permissions", [])
 
     dao_add_user_to_service(service, user, permissions, folder_permissions)
+    # TODO REMOVE DEBUG
+    print(hilite(f"ADDED {user.name} to service {service.name}"))
+    # END DEBUG
+
     data = service_schema.dump(service)
     return jsonify(data=data), 201
 

app/service_invite/rest.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import os
 from datetime import datetime
@@ -25,6 +26,7 @@ from app.notifications.process_notifications import (
     send_notification_to_queue,
 )
 from app.schemas import invited_user_schema
+from app.utils import hilite
 
 service_invite = Blueprint("service_invite", __name__)
 
@@ -32,6 +34,10 @@ register_errors(service_invite)
 
 
 def _create_service_invite(invited_user, invite_link_host):
+    # TODO REMOVE DEBUG
+    print(hilite("ENTER _create_service_invite"))
+    # END DEBUG
+
     template_id = current_app.config["INVITATION_EMAIL_TEMPLATE_ID"]
 
     template = dao_get_template_by_id(template_id)
@@ -43,9 +49,25 @@ def _create_service_invite(invited_user, invite_link_host):
         current_app.config["SECRET_KEY"],
         current_app.config["DANGEROUS_SALT"],
     )
+
+    # The raw permissions are in the form "a,b,c,d"
+    # but need to be in the form ["a", "b", "c", "d"]
+    data = {}
+    permissions = invited_user.permissions
+    permissions = permissions.split(",")
+    data["from_user_id"] = (str(invited_user.from_user.id))
+    data["service_id"] = str(invited_user.service.id)
+    data["permissions"] = permissions
+    data["folder_permissions"] = invited_user.folder_permissions
+    data["invited_user_id"] = str(invited_user.id)
+    data["invited_user_email"] = invited_user.email_address
+
     url = os.environ["LOGIN_DOT_GOV_REGISTRATION_URL"]
     url = url.replace("NONCE", token)
-    url = url.replace("STATE", token)
+
+    user_data_url_safe = get_user_data_url_safe(data)
+
+    url = url.replace("STATE", user_data_url_safe)
 
     personalisation = {
         "user_name": invited_user.from_user.name,
@@ -70,26 +92,6 @@ def _create_service_invite(invited_user, invite_link_host):
         json.dumps(personalisation),
         ex=1800,
     )
-    # The raw permissions are in the form "a,b,c,d"
-    # but need to be in the form ["a", "b", "c", "d"]
-    data = {}
-    permissions = invited_user.permissions
-    permissions = permissions.split(",")
-    permission_list = []
-    for permission in permissions:
-        permission_list.append(f"{permission}")
-    data["from_user_id"] = (str(invited_user.from_user.id),)
-    data["service_id"] = str(invited_user.service.id)
-    data["permissions"] = permission_list
-    data["folder_permissions"] = invited_user.folder_permissions
-
-    # This is for the login.gov service invite on the
-    # "Set Up Your Profile" path.
-    redis_store.set(
-        f"service-invite-{invited_user.email_address}",
-        json.dumps(data),
-        ex=3600 * 24,
-    )
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
 
 
@@ -212,3 +214,9 @@ def validate_service_invitation_token(token):
 
     invited_user = get_invited_user_by_id(invited_user_id)
     return jsonify(data=invited_user_schema.dump(invited_user)), 200
+
+
+def get_user_data_url_safe(data):
+    data = json.dumps(data)
+    data = base64.b64encode(data.encode("utf8"))
+    return data.decode("utf8")

poetry.lock

@@ -2490,6 +2490,7 @@ files = [
     {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
     {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
     {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
+    {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
     {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
 ]
 
@@ -3531,7 +3532,6 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},

terraform/README.md

@@ -89,17 +89,19 @@ These steps assume shared [Terraform state credentials](#terraform-state-credent
 
 1. Run `cf spaces` and, from the output, copy the space name for the environment you are working in, such as `notify-sandbox`.
 
-1. Next you will set up a SpaceDeployer. Prepare to fill in these values:
-   * `<SPACE_NAME>` will be the string you copied from the prior step
-   * `<ACCOUNT_NAME>` can be anything, although we recommend something that communicates the purpose of the deployer. For example: "circleci-deployer" for the credentials CircleCI uses to deploy the application, or "sandbox-<your_name>" for credentials to run terraform manually.
+1. Next you will set up a SpaceDeployer service account instance. This is something like a stub user account, just for deployment. Note these two values which you will use both to create and destroy the account:
+    1. `<SPACE_NAME>` will be the string you copied from the prior step
+    1. `<ACCOUNT_NAME>` can be anything, although we recommend something that communicates the purpose of the deployer. For example: "circleci-deployer" for the credentials CircleCI uses to deploy the application, or "sandbox-<your_name>" for credentials to run terraform manually.
 
-   Put those two values into this command:
+    Put those two values into this command:
     ```bash
-    ./create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME> > secrets.auto.tfvars
+    ../create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME> > secrets.auto.tfvars
     ```
 
     The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your `<ACCOUNT_NAME>`. The [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/) has more information.
 
+    Some resources you might work on require a SpaceDeployer account with higher permissions. Add the `-m` flag to the command to get this.
+
     The command uses the redirection operator (`>`) to write that output to the `secrets.auto.tfvars` file. Terraform will find the username and password there, and use them as input variables.
 
 1. While still in an environment directory, initialize Terraform:
@@ -137,6 +139,8 @@ These steps assume shared [Terraform state credentials](#terraform-state-credent
     ./destroy_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME>
     ```
 
+    List `cf services` if you are unsure which space deployer service instances still exist
+
     Optionally, you can also `rm secrets.auto.tfvars`
 
 ## Structure
@@ -195,3 +199,18 @@ You need to re-authenticate with the Cloud Foundry CLI
 cf login -a api.fr.cloud.gov --sso
 ```
 You may also need to log in again to the Cloud.gov website.
+
+### CF account not authorized
+
+```
+Error: You are not authorized to perform the requested action
+```
+This error indicates that the Cloud Foundry user account (or service account) needs OrgManager permissions to take the action.
+* When you create a SpaceDeployer service account, use the `-m` flag when running the `./create_service_account.sh` script
+* Your own CF user may may also require OrgManager permissions to run the script
+
+### Services limit
+```
+You have exceeded your organization's services limit.
+```
+Too many Cloud Foundry services have been created without being destroyed. Perhaps Terraform developers have forgotten to delete their SpaceDeployers after they finish with them. List `cf services` to see.

terraform/bootstrap/main.tf

@@ -3,7 +3,7 @@ locals {
 }
 
 module "s3" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.9.1"
 
   cf_org_name   = "gsa-tts-benefits-studio"
   cf_space_name = "notify-management"

terraform/bootstrap/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/demo/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/development/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/production/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/sandbox/main.tf

@@ -17,7 +17,7 @@ module "database" {
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.7.1"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.9.1"
 
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
@@ -27,7 +27,7 @@ module "redis" {
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.7.1"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.9.1"
 
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name

terraform/sandbox/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

terraform/shared/egress_space/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/shared/ses/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/shared/sns/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 }

terraform/staging/providers.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 1.0"
+  required_version = "~> 1.7"
   required_providers {
     cloudfoundry = {
       source  = "cloudfoundry-community/cloudfoundry"
-      version = "0.53.0"
+      version = "0.53.1"
     }
   }
 

tests/app/service_invite/test_service_invite_rest.py

@@ -31,6 +31,9 @@ def test_create_invited_user(
     extra_args,
     expected_start_of_invite_url,
 ):
+    mocker.patch("app.service_invite.rest.redis_store.raw_set")
+    mocker.patch("app.service_invite.rest.redis_store.raw_get")
+
     mocked = mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
     email_address = "invited_user@service.gov.uk"
     invite_from = sample_service.users[0]
@@ -92,6 +95,9 @@ def test_create_invited_user(
 def test_create_invited_user_without_auth_type(
     admin_request, sample_service, mocker, invitation_email_template
 ):
+
+    mocker.patch("app.service_invite.rest.redis_store.raw_set")
+    mocker.patch("app.service_invite.rest.redis_store.raw_get")
     mocker.patch("app.celery.provider_tasks.deliver_email.apply_async")
     email_address = "invited_user@service.gov.uk"
     invite_from = sample_service.users[0]
@@ -213,6 +219,9 @@ def test_resend_expired_invite(
     invitation_email_template,
     mocker,
 ):
+
+    mocker.patch("app.service_invite.rest.redis_store.raw_set")
+    mocker.patch("app.service_invite.rest.redis_store.raw_get")
     url = f"/service/{sample_expired_user.service_id}/invite/{sample_expired_user.id}/resend"
     mock_send = mocker.patch("app.service_invite.rest.send_notification_to_queue")
     mock_persist = mocker.patch("app.service_invite.rest.persist_notification")