From a3f594693ff91417388f41f0a5c7c3294d806e82 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 4 Nov 2022 11:48:01 -0400 Subject: [PATCH 1/7] Add sandbox space --- app/config.py | 1 + deploy-config/sandbox.yml | 10 +++++++ terraform/sandbox/main.tf | 55 ++++++++++++++++++++++++++++++++++ terraform/sandbox/providers.tf | 17 +++++++++++ terraform/sandbox/variables.tf | 4 +++ 5 files changed, 87 insertions(+) create mode 100644 deploy-config/sandbox.yml create mode 100644 terraform/sandbox/main.tf create mode 100644 terraform/sandbox/providers.tf create mode 100644 terraform/sandbox/variables.tf diff --git a/app/config.py b/app/config.py index 13e41d93e..d8808b04c 100644 --- a/app/config.py +++ b/app/config.py @@ -425,5 +425,6 @@ configs = { 'test': Test, 'staging': Staging, 'demo': Demo, + 'sandbox': Staging, 'production': Production } diff --git a/deploy-config/sandbox.yml b/deploy-config/sandbox.yml new file mode 100644 index 000000000..de1490b5c --- /dev/null +++ b/deploy-config/sandbox.yml @@ -0,0 +1,10 @@ +env: sandbox +web_instances: 1 +web_memory: 1G +worker_instances: 1 +worker_memory: 512M +public_api_route: notify-api-sandbox.app.cloud.gov +admin_base_url: https://notify-sandbox.app.cloud.gov +ADMIN_CLIENT_SECRET: dev-notify-secret-key +DANGEROUS_SALT: dev-notify-salt +SECRET_KEY: dev-notify-secret-key diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf new file mode 100644 index 000000000..f04d15b67 --- /dev/null +++ b/terraform/sandbox/main.tf @@ -0,0 +1,55 @@ +locals { + cf_org_name = "gsa-tts-benefits-studio-prototyping" + cf_space_name = "notify-sandbox" + env = "sandbox" + app_name = "notify-api" + recursive_delete = true +} + +module "database" { + source = "github.com/18f/terraform-cloudgov//database" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + env = local.env + app_name = local.app_name + recursive_delete = local.recursive_delete + rds_plan_name = "micro-psql" +} + +module "redis" { + source = "github.com/18f/terraform-cloudgov//redis" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + env = local.env + app_name = local.app_name + recursive_delete = local.recursive_delete + redis_plan_name = "redis-dev" +} + +module "csv_upload_bucket" { + source = "github.com/18f/terraform-cloudgov//s3" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + recursive_delete = local.recursive_delete + s3_service_name = "${local.app_name}-csv-upload-bucket-${local.env}" +} + +module "contact_list_bucket" { + source = "github.com/18f/terraform-cloudgov//s3" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + recursive_delete = local.recursive_delete + s3_service_name = "${local.app_name}-contact-list-bucket-${local.env}" +} diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf new file mode 100644 index 000000000..0cab42f6f --- /dev/null +++ b/terraform/sandbox/providers.tf @@ -0,0 +1,17 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "0.15.5" + } + } + + backend "s3" { + bucket = "cg-6b759c13-6253-4a64-9bda-dd1f620185b0" + key = "api.tfstate.sandbox" + encrypt = "true" + region = "us-gov-west-1" + profile = "notify-terraform-backend" + } +} diff --git a/terraform/sandbox/variables.tf b/terraform/sandbox/variables.tf new file mode 100644 index 000000000..a24f2f3f8 --- /dev/null +++ b/terraform/sandbox/variables.tf @@ -0,0 +1,4 @@ +variable "cf_password" { + sensitive = true +} +variable "cf_user" {} From 13134ac6ae2cfc5f926f73f874af8eea9cf183ab Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Fri, 4 Nov 2022 16:37:32 -0400 Subject: [PATCH 2/7] Add egress-space terraform module --- terraform/create_service_account.sh | 24 +++++++++++---- terraform/sandbox/main.tf | 13 ++++++++ terraform/shared/egress_space/main.tf | 36 ++++++++++++++++++++++ terraform/shared/egress_space/providers.tf | 16 ++++++++++ terraform/shared/egress_space/variables.tf | 10 ++++++ terraform/staging/main.tf | 12 ++++++++ 6 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 terraform/shared/egress_space/main.tf create mode 100644 terraform/shared/egress_space/providers.tf create mode 100644 terraform/shared/egress_space/variables.tf diff --git a/terraform/create_service_account.sh b/terraform/create_service_account.sh index fafe83adf..d69a90796 100755 --- a/terraform/create_service_account.sh +++ b/terraform/create_service_account.sh @@ -7,14 +7,18 @@ $0: Create a Service User Account for a given space Usage: $0 -h - $0 -s -u [-r ] [-o ] + $0 -s -u [-r ] [-o ] [-m] Options: -h: show help and exit -s : configure the space to act on. Required -u : set the service user name. Required -r : set the service user's role to either space-deployer or space-auditor. Default: space-deployer +-m: If provided, make the service user an OrgManager -o : configure the organization to act on. Default: $org + +Notes: +OrgManager is required for terraform to create -egress spaces " set -e @@ -23,8 +27,9 @@ set -o pipefail space="" service="" role="space-deployer" +org_manager="false" -while getopts ":hs:u:r:o:" opt; do +while getopts ":hms:u:r:o:" opt; do case "$opt" in s) space=${OPTARG} @@ -38,6 +43,9 @@ while getopts ":hs:u:r:o:" opt; do o) org=${OPTARG} ;; + m) + org_manager="true" + ;; h) echo "$usage" exit 0 @@ -60,13 +68,17 @@ cf create-service-key $service service-account-key 1>&2 # output service key to stdout in secrets.auto.tfvars format creds=`cf service-key $service service-account-key | tail -n 4` -username=`echo $creds | jq '.username'` -password=`echo $creds | jq '.password'` +username=`echo $creds | jq -r '.username'` +password=`echo $creds | jq -r '.password'` + +if [[ $org_manager = "true" ]]; then + cf set-org-role $username $org OrgManager 1>&2 +fi cat << EOF # generated with $0 -s $space -u $service -r $role -o $org # revoke with $(dirname $0)/destroy_service_account.sh -s $space -u $service -o $org -cf_user = $username -cf_password = $password +cf_user = "$username" +cf_password = "$password" EOF diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf index f04d15b67..fd1c4ced3 100644 --- a/terraform/sandbox/main.tf +++ b/terraform/sandbox/main.tf @@ -53,3 +53,16 @@ module "contact_list_bucket" { recursive_delete = local.recursive_delete s3_service_name = "${local.app_name}-contact-list-bucket-${local.env}" } + +module "egress-space" { + source = "../shared/egress_space" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_restricted_space_name = local.cf_space_name + deployers = [ + var.cf_user, + "ryan.ahearn@gsa.gov" + ] +} diff --git a/terraform/shared/egress_space/main.tf b/terraform/shared/egress_space/main.tf new file mode 100644 index 000000000..4a65b74e9 --- /dev/null +++ b/terraform/shared/egress_space/main.tf @@ -0,0 +1,36 @@ +### +# Target space/org +### + +data "cloudfoundry_org" "org" { + name = var.cf_org_name +} + +### +# Egress Space +### + +resource "cloudfoundry_space" "public_egress" { + name = "${var.cf_restricted_space_name}-egress" + org = data.cloudfoundry_org.org.id +} + +### +# User roles +### + +data "cloudfoundry_user" "users" { + for_each = var.deployers + name = each.key + org_id = data.cloudfoundry_org.org.id +} + +locals { + user_ids = [for user in data.cloudfoundry_user.users : user.id] +} + +resource "cloudfoundry_space_users" "deployers" { + space = cloudfoundry_space.public_egress.id + managers = local.user_ids + developers = local.user_ids +} diff --git a/terraform/shared/egress_space/providers.tf b/terraform/shared/egress_space/providers.tf new file mode 100644 index 000000000..ad8addecd --- /dev/null +++ b/terraform/shared/egress_space/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "~> 0.15" + } + } +} + +provider "cloudfoundry" { + api_url = "https://api.fr.cloud.gov" + user = var.cf_user + password = var.cf_password + app_logs_max = 30 +} diff --git a/terraform/shared/egress_space/variables.tf b/terraform/shared/egress_space/variables.tf new file mode 100644 index 000000000..1ab080dea --- /dev/null +++ b/terraform/shared/egress_space/variables.tf @@ -0,0 +1,10 @@ +variable "cf_password" { + type = string + sensitive = true +} +variable "cf_user" {} +variable "cf_org_name" {} +variable "cf_restricted_space_name" {} +variable "deployers" { + type = set(string) +} diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index abbb0a5be..6fca24d76 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -53,3 +53,15 @@ module "contact_list_bucket" { recursive_delete = local.recursive_delete s3_service_name = "${local.app_name}-contact-list-bucket-${local.env}" } + +module "egress-space" { + source = "../shared/egress_space" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_restricted_space_name = local.cf_space_name + deployers = [ + var.cf_user + ] +} From 898a570f8ffd5031a3384314ea97842dc48149fb Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 8 Nov 2022 11:48:23 -0500 Subject: [PATCH 3/7] Set https_proxy env variable in app initialization script --- .profile | 6 ++++++ manifest.yml | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 .profile diff --git a/.profile b/.profile new file mode 100644 index 000000000..9960f33a2 --- /dev/null +++ b/.profile @@ -0,0 +1,6 @@ +## +# Cloud Foundry app initialization script +# https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile +## + +export https_proxy=$egress_proxy diff --git a/manifest.yml b/manifest.yml index d032ed1bf..0df7a50b1 100644 --- a/manifest.yml +++ b/manifest.yml @@ -43,3 +43,5 @@ applications: AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 + + REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" From 3020d3d94e3d78bb829de0ca89abe0ae5ec27959 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 8 Nov 2022 13:51:39 -0500 Subject: [PATCH 4/7] Add github action for deploying egress proxy --- .github/actions/deploy-proxy/action.yml | 33 +++++++++++++++++++ .github/workflows/deploy-demo.yml | 6 ++++ .github/workflows/deploy.yml | 6 ++++ .../egress_proxy/notify-api-demo.allow.acl | 3 ++ .../egress_proxy/notify-api-demo.deny.acl | 0 .../egress_proxy/notify-api-staging.allow.acl | 3 ++ .../egress_proxy/notify-api-staging.deny.acl | 0 7 files changed, 51 insertions(+) create mode 100644 .github/actions/deploy-proxy/action.yml create mode 100644 deploy-config/egress_proxy/notify-api-demo.allow.acl create mode 100644 deploy-config/egress_proxy/notify-api-demo.deny.acl create mode 100644 deploy-config/egress_proxy/notify-api-staging.allow.acl create mode 100644 deploy-config/egress_proxy/notify-api-staging.deny.acl diff --git a/.github/actions/deploy-proxy/action.yml b/.github/actions/deploy-proxy/action.yml new file mode 100644 index 000000000..d301aa40a --- /dev/null +++ b/.github/actions/deploy-proxy/action.yml @@ -0,0 +1,33 @@ +name: Deploy egress proxy +description: Setup egres space and deploy proxy +inputs: + cf_space: + description: The space the target app exists in. + required: true + app: + description: application name to be proxied. + required: true +runs: + using: composite + steps: + - name: Set restricted space egress + shell: bash + run: ./terraform/set_space_egress.sh -t -s ${{ inputs.cf_space }} + - name: Set public space egress + shell: bash + run: ./terraform/set_space_egress.sh -p -s ${{ inputs.cf_space }}-egress + - name: Create temp directory + shell: bash + id: create-temp-dir + run: echo "path=$(mktemp -d -t egress-XXXXXXXXXX --tmpdir=$RUNNER_TEMP)" >> $GITHUB_OUTPUT + - name: Clone egress-proxy + shell: bash + run: git clone https://github.com/rahearn/cg-egress-proxy.git ${{ steps.create-temp-dir.outputs.path }} + - name: Copy config files + shell: bash + run: cp ./deploy-config/egress_proxy/${{ inputs.app }}.*.acl ${{ steps.create-temp-dir.outputs.path }} + - name: Deploy proxy + shell: bash + run: > + cd ${{ steps.create-temp-dir.outputs.path }}; + ./bin/cf-deployproxy -a ${{ inputs.app }} -p egress-proxy -e egress_proxy diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 813ca0c19..c973d6e89 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -65,3 +65,9 @@ jobs: --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" + + - name: Deploy egress proxy + uses: ./.github/actions/deploy-proxy + with: + cf_space: notify-demo + app: notify-api-demo diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 50c423ea3..0e8f29a7b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -71,6 +71,12 @@ jobs: --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" + - name: Deploy egress proxy + uses: ./.github/actions/deploy-proxy + with: + cf_space: notify-staging + app: notify-api-staging + bail: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'failure' }} diff --git a/deploy-config/egress_proxy/notify-api-demo.allow.acl b/deploy-config/egress_proxy/notify-api-demo.allow.acl new file mode 100644 index 000000000..59ba51ac1 --- /dev/null +++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl @@ -0,0 +1,3 @@ +email.us-west-2.amazonaws.com +sns.us-west-2.amazonaws.com +logs.us-west-2.amazonaws.com diff --git a/deploy-config/egress_proxy/notify-api-demo.deny.acl b/deploy-config/egress_proxy/notify-api-demo.deny.acl new file mode 100644 index 000000000..e69de29bb diff --git a/deploy-config/egress_proxy/notify-api-staging.allow.acl b/deploy-config/egress_proxy/notify-api-staging.allow.acl new file mode 100644 index 000000000..59ba51ac1 --- /dev/null +++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl @@ -0,0 +1,3 @@ +email.us-west-2.amazonaws.com +sns.us-west-2.amazonaws.com +logs.us-west-2.amazonaws.com diff --git a/deploy-config/egress_proxy/notify-api-staging.deny.acl b/deploy-config/egress_proxy/notify-api-staging.deny.acl new file mode 100644 index 000000000..e69de29bb From 2c140409df84330f266f92fd078395ae5fc6da0f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 8 Nov 2022 14:57:35 -0500 Subject: [PATCH 5/7] Fix typo --- .github/actions/deploy-proxy/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy-proxy/action.yml b/.github/actions/deploy-proxy/action.yml index d301aa40a..56e87080f 100644 --- a/.github/actions/deploy-proxy/action.yml +++ b/.github/actions/deploy-proxy/action.yml @@ -1,5 +1,5 @@ name: Deploy egress proxy -description: Setup egres space and deploy proxy +description: Set egress space security groups and deploy proxy inputs: cf_space: description: The space the target app exists in. From 82847d3f717585dd0d349bd6fb029d2f909572ac Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 8 Nov 2022 16:11:15 -0500 Subject: [PATCH 6/7] Only deploy egress proxy if the config changed --- .github/workflows/deploy-demo.yml | 8 +++++++- .github/workflows/deploy.yml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index c973d6e89..592228422 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -18,7 +18,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v1.1.2 + uses: tj-actions/changed-files@v34 with: files: terraform/demo - name: Terraform init @@ -66,7 +66,13 @@ jobs: --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" + - name: Check for changes to egress config + id: changed-egress-config + uses: tj-actions/changed-files@v34 + with: + files: deploy-config/egress_proxy/notify-api-demo.*.acl - name: Deploy egress proxy + if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy with: cf_space: notify-demo diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0e8f29a7b..dd9f6255d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,7 +23,7 @@ jobs: - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v1.1.2 + uses: tj-actions/changed-files@v34 with: files: terraform/staging - name: Terraform init @@ -71,7 +71,13 @@ jobs: --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" + - name: Check for changes to egress config + id: changed-egress-config + uses: tj-actions/changed-files@v34 + with: + files: deploy-config/egress_proxy/notify-api-staging.*.acl - name: Deploy egress proxy + if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy with: cf_space: notify-staging From 2e6deb1556269ed564921704aa9d57dd26a2ccb1 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 8 Nov 2022 16:26:59 -0500 Subject: [PATCH 7/7] Document egress proxy and sandbox environment within deploying.md --- docs/deploying.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/deploying.md b/docs/deploying.md index ff1041b91..15c41afbc 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -20,3 +20,42 @@ Configurations for these are located in [the `deploy-config` folder](../deploy-c In the event that a deployment includes a Terraform change, that change will run before any code is deployed to the environment. Each environment has its own Terraform GitHub Action to handle that change. Failures in any of these GitHub workflows will be surfaced in the Pull Request related to the code change, and in the case of `checks.yml` actively prevent the PR from being merged. Failure in the Terraform workflow will not actively prevent the PR from being merged, but reviewers should not approve a PR with a failing terraform plan. + +## Egress Proxy + +The API app runs in a [restricted egress space](https://cloud.gov/docs/management/space-egress/). +This allows direct communication to cloud.gov-brokered services, but +not to other APIs that we require. + +As part of the deploy, we create an +[egress proxy application](https://github.com/GSA/cg-egress-proxy) that allows traffic out of our +application to a select list of allowed domains. + +Update the allowed domains by updating `deploy-config/egress_proxy/notify-api-.allow.acl` +and deploying an updated version of the application throught he normal deploy process. + +## Sandbox environment + +There is a sandbox space, complete with terraform and `deploy-config/sandbox.yml` file available +for experimenting with infrastructure changes without going through the full CI/CD cycle each time. + +Rules for use: + +1. Ensure that no other developer is using the environment, as there is nothing stopping changes from overwriting each other. +1. Clean up services you create when you are done. `terraform destroy` from within the `terraform/sandbox` directory should do it. + +### Deploying to the sandbox + +1. Set up services: + ``` + $ cd terraform/sandbox + $ ../create_service_account.sh -s notify-sandbox -u -terraform -m > secrets.auto.tfvars + $ terraform init + $ terraform plan + $ terraform apply + ``` +1. start a pipenv shell as a shortcut to load `.env` file variables: `$ pipenv shell` +1. Deploy the application: + ``` + cf push --vars-file deploy-config/sandbox.yml --var AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID --var AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + ```