diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3c7f90d9b..1efb81a27 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,6 +23,30 @@ jobs: libcurl4-openssl-dev - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Check for changes to Terraform + id: changed-terraform-files + uses: tj-actions/changed-files@v1.1.2 + with: + files: terraform/staging + - name: Terraform init + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + run: terraform init + - name: Terraform apply + if: steps.changed-terraform-files.outputs.any_changed == 'true' + working-directory: terraform/staging + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} + TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + run: terraform apply -auto-approve -input=false - name: Set up Python 3.9 uses: actions/setup-python@v3 @@ -48,6 +72,7 @@ jobs: cf_org: gsa-10x-prototyping cf_space: 10x-notifications push_arguments: >- + --var env=staging --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" diff --git a/.github/workflows/terraform-production.yml b/.github/workflows/terraform-production.yml new file mode 100644 index 000000000..7861c3205 --- /dev/null +++ b/.github/workflows/terraform-production.yml @@ -0,0 +1,79 @@ +name: Run Terraform plan in production + +on: + pull_request: + branches: [ production ] + paths: [ 'terraform/**' ] + +defaults: + run: + working-directory: terraform/production + +jobs: + terraform: + name: Terraform plan + runs-on: ubuntu-latest + environment: production + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Terraform format + id: format + run: terraform fmt -check + + - name: Terraform init + id: init + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + run: terraform init + + - name: Terraform validate + id: validation + run: terraform validate -no-color + + - name: Terraform plan + id: plan + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + TF_VAR_cf_user: ${{ secrets.CF_USERNAME }} + TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }} + run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt + + - name: Read Terraform plan output file + id: terraform_output + uses: juliangruber/read-file-action@v1 + if: ${{ always() }} + with: + path: ./terraform/production/plan_output.txt + + # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow + - name: Update PR + uses: actions/github-script@v4 + # we would like to update the PR even when a prior step failed + if: ${{ always() }} + with: + script: | + const output = `Terraform Format and Style: ${{ steps.format.outcome }} + Terraform Initialization: ${{ steps.init.outcome }} + Terraform Validation: ${{ steps.validation.outcome }} + Terraform Plan: ${{ steps.plan.outcome }} + +
Show Plan + + \`\`\`\n + ${{ steps.terraform_output.outputs.content }} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) diff --git a/.github/workflows/terraform-staging.yml b/.github/workflows/terraform-staging.yml new file mode 100644 index 000000000..5c7d2a6ff --- /dev/null +++ b/.github/workflows/terraform-staging.yml @@ -0,0 +1,78 @@ +name: Run Terraform plan in staging + +on: + pull_request: + branches: [ main ] + paths: [ 'terraform/**' ] + +defaults: + run: + working-directory: terraform/staging + +jobs: + terraform: + name: Terraform plan + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Terraform format + id: format + run: terraform fmt -check + + - name: Terraform init + id: init + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + run: terraform init + + - name: Terraform validate + id: validation + run: terraform validate -no-color + + - name: Terraform plan + id: plan + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} + TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }} + TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt + + - name: Read Terraform plan output file + id: terraform_output + uses: juliangruber/read-file-action@v1 + if: ${{ always() }} + with: + path: ./terraform/staging/plan_output.txt + + # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow + - name: Update PR + uses: actions/github-script@v4 + # we would like to update the PR even when a prior step failed + if: ${{ always() }} + with: + script: | + const output = `Terraform Format and Style: ${{ steps.format.outcome }} + Terraform Initialization: ${{ steps.init.outcome }} + Terraform Validation: ${{ steps.validation.outcome }} + Terraform Plan: ${{ steps.plan.outcome }} + +
Show Plan + + \`\`\`\n + ${{ steps.terraform_output.outputs.content }} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) diff --git a/.gitignore b/.gitignore index 7fca645b5..3d58154fe 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,10 @@ logs/** app/version.py # app/static # is generated at build time, TODO: de-UKify and ignore again # app/templates/vendor # is generated at build time, TODO: de-UKify and ignore again + +# Terraform +.terraform.lock.hcl +**/.terraform/* +secrets.auto.tfvars +terraform.tfstate +terraform.tfstate.backup diff --git a/manifest.yml b/manifest.yml index ba66d920f..b0e658864 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,6 +1,6 @@ --- applications: - - name: notifications-admin + - name: notifications-admin-((env)) buildpack: python_buildpack memory: 1G health-check-type: http @@ -13,7 +13,7 @@ applications: # - logit-ssl-syslog-drain # - notify-prometheus # - notify-splunk - - api-redis + - notifications-admin-redis-((env)) env: NOTIFY_APP_NAME: admin diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 000000000..a812b2208 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,130 @@ +# Terraform + +This directory holds the terraform modules for maintaining your complete persistent infrastructure. + +Prerequisite: install the `jq` JSON processor: `brew install jq` + +## Initial setup + +1. Manually run the bootstrap module following instructions under `Terraform State Credentials` +1. Setup CI/CD Pipeline to run Terraform + 1. Copy bootstrap credentials to your CI/CD secrets using the instructions in the base README + 1. Create a cloud.gov SpaceDeployer by following the instructions under `SpaceDeployers` + 1. Copy SpaceDeployer credentials to your CI/CD secrets using the instructions in the base README +1. Manually Running Terraform + 1. Follow instructions under `Set up a new environment` to create your infrastructure + +## Terraform State Credentials + +The bootstrap module is used to create an s3 bucket for later terraform runs to store their state in. + +### Bootstrapping the state storage s3 buckets for the first time + +1. Run `terraform init` +1. Run `./run.sh plan` to verify that the changes are what you expect +1. Run `./run.sh apply` to set up the bucket and retrieve credentials +1. Follow instructions under `Use bootstrap credentials` +1. Ensure that `import.sh` includes a line and correct IDs for any resources created +1. Run `./teardown_creds.sh` to remove the space deployer account used to create the s3 bucket + +### To make changes to the bootstrap module + +*This should not be necessary in most cases* + +1. Run `terraform init` +1. If you don't have terraform state locally: + 1. run `./import.sh` + 1. optionally run `./run.sh apply` to include the existing outputs in the state file +1. Make your changes +1. Continue from step 2 of the boostrapping instructions + +### Retrieving existing bucket credentials + +1. Run `./run.sh show` +1. Follow instructions under `Use bootstrap credentials` + +#### Use bootstrap credentials + +1. Add the following to `~/.aws/credentials` + ``` + [notify-terraform-backend] + aws_access_key_id = + aws_secret_access_key = + ``` + +1. Copy `bucket` from `bucket_credentials` output to the backend block of `staging/providers.tf` and `production/providers.tf` + +## SpaceDeployers + +A [SpaceDeployer](https://cloud.gov/docs/services/cloud-gov-service-account/) account is required to run terraform or +deploy the application from the CI/CD pipeline. Create a new account by running: + +`./create_service_account.sh -s -u ` + +## Set up a new environment manually + +The below steps rely on you first configuring access to the Terraform state in s3 as described in [Terraform State Credentials](#terraform-state-credentials). + +1. `cd` to the environment you are working in + +1. Set up a SpaceDeployer + ```bash + # create a space deployer service instance that can log in with just a username and password + # the value of < SPACE_NAME > should be `staging` or `prod` depending on where you are working + # the value for < ACCOUNT_NAME > can be anything, although we recommend + # something that communicates the purpose of the deployer + # for example: circleci-deployer for the credentials CircleCI uses to + # deploy the application or -terraform for credentials to run terraform manually + ./create_service_account.sh -s -u > secrets.auto.tfvars + ``` + + The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your ``. Read more in the [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/). + + The easiest way to use this script is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in + +1. Run terraform from your new environment directory with + ```bash + terraform init + terraform plan + ``` + +1. Apply changes with `terraform apply`. + +1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform once. + ```bash + # and have the same values as used above. + ./destroy_service_account.sh -s -u + ``` + +## Structure + +Each environment has its own module, which relies on a shared module for everything except the providers code and environment specific variables and settings. + +``` +- bootstrap/ + |- main.tf + |- providers.tf + |- variables.tf + |- run.sh + |- teardown_creds.sh + |- import.sh +- / + |- main.tf + |- providers.tf + |- secrets.auto.tfvars + |- variables.tf +``` + +In the environment-specific modules: +- `providers.tf` lists the required providers +- `main.tf` calls the shared Terraform code, but this is also a place where you can add any other services, resources, etc, which you would like to set up for that environment +- `variables.tf` lists the variables that will be needed, either to pass through to the child module or for use in this module +- `secrets.auto.tfvars` is a file which contains the information about the service-key and other secrets that should not be shared + +In the bootstrap module: +- `providers.tf` lists the required providers +- `main.tf` sets up s3 bucket to be shared across all environments. It lives in `prod` to communicate that it should not be deleted +- `variables.tf` lists the variables that will be needed. Most values are hard-coded in this module +- `run.sh` Helper script to set up a space deployer and run terraform. The terraform action (`show`/`plan`/`apply`/`destroy`) is passed as an argument +- `teardown_creds.sh` Helper script to remove the space deployer setup as part of `run.sh` +- `import.sh` Helper script to create a new local state file in case terraform changes are needed diff --git a/terraform/bootstrap/import.sh b/terraform/bootstrap/import.sh new file mode 100755 index 000000000..88b1e40d2 --- /dev/null +++ b/terraform/bootstrap/import.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +read -p "Are you sure you want to import terraform state (y/n)? " verify + +if [[ $verify == "y" ]]; then + echo "Importing bootstrap state" + ./run.sh import module.s3.cloudfoundry_service_instance.bucket 31204bcc-aae3-4cd3-8b59-5055a338d44f + ./run.sh import cloudfoundry_service_key.bucket_creds 483a6ac5-4ba0-48ad-9850-ef87b51aaa08 + ./run.sh plan +else + echo "Not importing bootstrap state" +fi diff --git a/terraform/bootstrap/main.tf b/terraform/bootstrap/main.tf new file mode 100644 index 000000000..f00fff4c5 --- /dev/null +++ b/terraform/bootstrap/main.tf @@ -0,0 +1,24 @@ +locals { + cf_api_url = "https://api.fr.cloud.gov" + s3_service_name = "notify-terraform-state" +} + +module "s3" { + source = "github.com/18f/terraform-cloudgov//s3" + + cf_api_url = local.cf_api_url + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = "gsa-10x-prototyping" + cf_space_name = "10x-notifications" + s3_service_name = local.s3_service_name +} + +resource "cloudfoundry_service_key" "bucket_creds" { + name = "${local.s3_service_name}-access" + service_instance = module.s3.bucket_id +} + +output "bucket_credentials" { + value = cloudfoundry_service_key.bucket_creds.credentials +} diff --git a/terraform/bootstrap/providers.tf b/terraform/bootstrap/providers.tf new file mode 100644 index 000000000..44ccda085 --- /dev/null +++ b/terraform/bootstrap/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "0.15.5" + } + } +} + +provider "cloudfoundry" { + api_url = local.cf_api_url + user = var.cf_user + password = var.cf_password + app_logs_max = 30 +} diff --git a/terraform/bootstrap/run.sh b/terraform/bootstrap/run.sh new file mode 100755 index 000000000..404987590 --- /dev/null +++ b/terraform/bootstrap/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +if [[ ! -f "secrets.auto.tfvars" ]]; then + ../create_service_account.sh -s 10x-notifications -u config-bootstrap-deployer > secrets.auto.tfvars +fi + +if [[ $# -gt 0 ]]; then + echo "Running terraform $@" + terraform $@ +else + echo "Not running terraform" +fi diff --git a/terraform/bootstrap/teardown_creds.sh b/terraform/bootstrap/teardown_creds.sh new file mode 100755 index 000000000..196e3756f --- /dev/null +++ b/terraform/bootstrap/teardown_creds.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +../destroy_service_account.sh -s 10x-notifications -u config-bootstrap-deployer + +rm secrets.auto.tfvars diff --git a/terraform/bootstrap/variables.tf b/terraform/bootstrap/variables.tf new file mode 100644 index 000000000..2fe500544 --- /dev/null +++ b/terraform/bootstrap/variables.tf @@ -0,0 +1,2 @@ +variable "cf_password" {} +variable "cf_user" {} diff --git a/terraform/create_service_account.sh b/terraform/create_service_account.sh new file mode 100755 index 000000000..1a6b0eb1c --- /dev/null +++ b/terraform/create_service_account.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +org="gsa-10x-prototyping" + +usage=" +$0: Create a Service User Account for a given space + +Usage: + $0 -h + $0 -s -u [-r ] [-o ] + +Options: +-h: show help and exit +-s : configure the space to act on. Required +-u : set the service user name. Required +-r : set the service user's role to either space-deployer or space-auditor. Default: space-deployer +-o : configure the organization to act on. Default: $org +" + +set -e +set -o pipefail + +space="" +service="" +role="space-deployer" + +while getopts ":hs:u:r:o:" opt; do + case "$opt" in + s) + space=${OPTARG} + ;; + u) + service=${OPTARG} + ;; + r) + role=${OPTARG} + ;; + o) + org=${OPTARG} + ;; + h) + echo "$usage" + exit 0 + ;; + esac +done + +if [[ $space = "" || $service = "" ]]; then + echo "$usage" + exit 1 +fi + +cf target -o $org -s $space 1>&2 + +# create user account service +cf create-service cloud-gov-service-account $role $service 1>&2 + +# create service key +cf create-service-key $service service-account-key 1>&2 + +# output service key to stdout in secrets.auto.tfvars format +creds=`cf service-key $service service-account-key | tail -n 4` +username=`echo $creds | jq '.username'` +password=`echo $creds | jq '.password'` + +cat << EOF +# generated with $0 -s $space -u $service -r $role -o $org +# revoke with $(dirname $0)/destroy_service_account.sh -s $space -u $service -o $org + +cf_user = $username +cf_password = $password +EOF diff --git a/terraform/destroy_service_account.sh b/terraform/destroy_service_account.sh new file mode 100755 index 000000000..caeb12901 --- /dev/null +++ b/terraform/destroy_service_account.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +org="gsa-10x-prototyping" + +usage=" +$0: Destroy a Service User Account in a given space + +Usage: + $0 -h + $0 -s -u [-o ] + +Options: +-h: show help and exit +-s : configure the space to act on. Required +-u : configure the service user name to destroy. Required +-o : configure the organization to act on. Default: $org +" + +set -e + +space="" +service="" + +while getopts ":hs:u:o:" opt; do + case "$opt" in + s) + space=${OPTARG} + ;; + u) + service=${OPTARG} + ;; + o) + org=${OPTARG} + ;; + h) + echo "$usage" + exit 0 + ;; + esac +done + +if [[ $space = "" || $service = "" ]]; then + echo "$usage" + exit 1 +fi + +cf target -o $org -s $space + +# destroy service key +cf delete-service-key $service service-account-key -f + +# destroy service +cf delete-service $service -f diff --git a/terraform/production/main.tf b/terraform/production/main.tf new file mode 100644 index 000000000..8f1b5574a --- /dev/null +++ b/terraform/production/main.tf @@ -0,0 +1,41 @@ +locals { + cf_org_name = "TKTK" + cf_space_name = "TKTK" + env = "production" + app_name = "notifications-admin" + recursive_delete = false +} + +module "redis" { + source = "github.com/18f/terraform-cloudgov//redis" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + env = local.env + app_name = local.app_name + recursive_delete = local.recursive_delete + redis_plan_name = "TKTK-production-redis-plan" +} + +########################################################################### +# The following lines need to be commented out for the initial `terraform apply` +# It can be re-enabled after: +# 1) the app has first been deployed +# 2) the route has been manually created by an OrgManager: +# `cf create-domain TKTK-org-name TKTK-production-domain-name` +########################################################################### +# module "domain" { +# source = "github.com/18f/terraform-cloudgov//domain" +# +# cf_user = var.cf_user +# cf_password = var.cf_password +# cf_org_name = local.cf_org_name +# cf_space_name = local.cf_space_name +# env = local.env +# app_name = local.app_name +# recursive_delete = local.recursive_delete +# cdn_plan_name = "domain" +# domain_name = "TKTK-production-domain-name" +# } diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf new file mode 100644 index 000000000..685d356f9 --- /dev/null +++ b/terraform/production/providers.tf @@ -0,0 +1,17 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "0.15.5" + } + } + + backend "s3" { + bucket = "cg-31204bcc-aae3-4cd3-8b59-5055a338d44f" + key = "admin.tfstate.prod" + encrypt = "true" + region = "us-gov-west-1" + profile = "notify-terraform-backend" + } +} diff --git a/terraform/production/variables.tf b/terraform/production/variables.tf new file mode 100644 index 000000000..2fe500544 --- /dev/null +++ b/terraform/production/variables.tf @@ -0,0 +1,2 @@ +variable "cf_password" {} +variable "cf_user" {} diff --git a/terraform/set_space_egress.sh b/terraform/set_space_egress.sh new file mode 100755 index 000000000..7eeaaf989 --- /dev/null +++ b/terraform/set_space_egress.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +org="gsa-10x-prototyping" + +usage=" +$0: Set egress rules for given space + +Usage: + $0 -h + $0 -s [-o ] [-p] [-t] + +Options: +-h: show help and exit +-s : configure the space to act on. Required +-o : configure the organization to act on. Default: $org +-p: Add the public egress rules +-t: Add the trusted egress rules + +Notes: +* If -p or -t are not passed, the related security groups will be removed, if they were present +" + +set -e + +space="" +public=false +trusted=false + +while getopts ":hs:o:pt" opt; do + case "$opt" in + s) + space=${OPTARG} + ;; + o) + org=${OPTARG} + ;; + p) + public=true + ;; + t) + trusted=true + ;; + h) + echo "$usage" + exit 0 + ;; + esac +done + +if [[ $space = "" ]]; then + echo "$usage" + exit 1 +fi + +if [[ $public = true ]]; then + cf bind-security-group public_networks_egress $org --space $space +else + cf unbind-security-group public_networks_egress $org $space +fi + +if [[ $trusted = true ]]; then + cf bind-security-group trusted_local_networks_egress $org --space $space +else + cf unbind-security-group trusted_local_networks_egress $org $space +fi + +echo "Done" diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf new file mode 100644 index 000000000..7d09ed8c0 --- /dev/null +++ b/terraform/staging/main.tf @@ -0,0 +1,20 @@ +locals { + cf_org_name = "gsa-10x-prototyping" + cf_space_name = "10x-notifications" + env = "staging" + app_name = "notifications-admin" + recursive_delete = true +} + +module "redis" { + source = "github.com/18f/terraform-cloudgov//redis" + + cf_user = var.cf_user + cf_password = var.cf_password + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + env = local.env + app_name = local.app_name + recursive_delete = local.recursive_delete + redis_plan_name = "redis-dev" +} diff --git a/terraform/staging/providers.tf b/terraform/staging/providers.tf new file mode 100644 index 000000000..826e74267 --- /dev/null +++ b/terraform/staging/providers.tf @@ -0,0 +1,17 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "0.15.5" + } + } + + backend "s3" { + bucket = "cg-31204bcc-aae3-4cd3-8b59-5055a338d44f" + key = "admin.tfstate.stage" + encrypt = "true" + region = "us-gov-west-1" + profile = "notify-terraform-backend" + } +} diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf new file mode 100644 index 000000000..2fe500544 --- /dev/null +++ b/terraform/staging/variables.tf @@ -0,0 +1,2 @@ +variable "cf_password" {} +variable "cf_user" {}