From 627149402c06e84af9c0d336c09a196c5eaa4b4a Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 13 Mar 2023 12:58:50 -0400 Subject: [PATCH 1/5] Add terraform/development for retrieving credentials for local development use --- docs/infra-overview.md | 61 ++++++++++++++------ sample.env | 32 +++++------ terraform/development/main.tf | 91 ++++++++++++++++++++++++++++++ terraform/development/providers.tf | 16 ++++++ terraform/development/run.sh | 69 ++++++++++++++++++++++ terraform/development/variables.tf | 5 ++ 6 files changed, 238 insertions(+), 36 deletions(-) create mode 100644 terraform/development/main.tf create mode 100644 terraform/development/providers.tf create mode 100755 terraform/development/run.sh create mode 100644 terraform/development/variables.tf diff --git a/docs/infra-overview.md b/docs/infra-overview.md index 09ed4df97..96727784b 100644 --- a/docs/infra-overview.md +++ b/docs/infra-overview.md @@ -36,24 +36,39 @@ In addition to terraform directories in the api and admin apps above: ## Terraform +### Development + +There are several remote services required for local development: + +* s3 +* ses +* sns + +Credentials for these services are created by running: + +1. `cd terraform/development` +1. `./run.sh` + +This will append credentials to your `.env` file. You will need to manually clean up any prior runs from that file if you run that command again. + +Offboarding: Service key bindings can be cleaned up from cloud.gov by running `./run.sh -d` yourself, or another developer running `./run.sh -d -u USER_TO_CLEANUP` + +### Cloud.gov + The cloud.gov environment is configured with Terraform. See [the `terraform` folder](../terraform/) to learn about that. ## AWS -In addition to services provisioned through cloud.gov, we have several services provisioned directly in AWS. Our AWS services are currently located in the us-west-2 region using the tts-sandbox account. We plan to move to GovCloud shortly. +In addition to services provisioned through cloud.gov, we have several services provisioned via [supplemental service brokers](https://github.com/GSA/usnotify-ssb) in AWS. Our AWS services are currently located in [several regions](https://github.com/GSA/usnotify-ssb#aws-accounts-and-regions-in-use) using Studio-controlled AWS accounts. To send messages, we use Amazon Web Services SNS and SES. In addition, we use AWS Pinpoint to provision and manage phone numbers, short codes, and long codes for sending SMS. -In SES, we are currently using the "sandbox" mode. This requires email addresses to be pre-registered in the AWS console in order to receive emails. The DKIM settings live under the verified domain entry. - In SNS, we have 3 topics for SMS receipts. These are not currently functional, so senders won't know the status of messages. Through Pinpoint, the API needs at least one number so that the application itself can send SMS for authentication codes. The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos. -SES and SNS for use by the cloud.gov-deployed apps is currently in the process of migrating to being provisioned through cloud.gov. Currently, SES, SNS, and S3 for local-development are still manually provisioned in AWS. - ## New Relic We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for application monitoring and error reporting. When requesting access to New Relic, ask to be added to the Benefits-Studio subaccount. @@ -62,32 +77,42 @@ We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for - [ ] Join [the GSA GitHub org](https://github.com/GSA/GitHub-Administration#join-the-gsa-organization) - [ ] Get permissions for the repos -- [ ] Get access to the cloud.gov org && space +- [ ] Get access to the cloud.gov org && spaces - [ ] Get [access to AWS](https://handbook.tts.gsa.gov/launching-software/infrastructure/#cloud-service-provider-csp-sandbox-accounts), if necessary - [ ] Get [access to New Relic](https://handbook.tts.gsa.gov/tools/new-relic/#how-do-i-get-access-to-new-relic), if necessary -- [ ] Pull down creds from cloud.gov and create the local .env file +- [ ] Create the local `.env` file by copying `sample.env` and running `./run.sh` within the `terraform/development` folder - [ ] Do stuff! ## Setting up the infrastructure +These steps are required for new cloud.gov environments. Local development borrows SES & SNS infrastructure from the `notify-staging` cloud.gov space, so these steps are not required for new developers. + ### Steps to prepare SES 1. After the first deploy of the application with the SSB-brokered SES service completes: 1. Log into the SES console and navigate to the SNS subscription page. - 2. Select "Request confirmation" for any subscriptions still in "Pending Confirmation" state -2. (For sandbox SES accounts) Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. -3. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. + 1. Select "Request confirmation" for any subscriptions still in "Pending Confirmation" state +1. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures ### Steps to prepare SNS -1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" -2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save -3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" -4. Under "Number Settings" choose "Request Phone Number" -5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" -6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" -7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) +#### Move SNS out of sandbox. -At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉 +1. Visit the SNS console for the region you will be sending from. Notes: + 1. SNS settings are per-region, so each environment must have its own region + 1. Pinpoint and SNS have confusing regional availability, so ensure both are available before submitting any requests. +1. Choose `Text messaging (SMS)` from the sidebar +1. Click the `Exit SMS Sandbox` button and submit the support request. This request should take at most a day to complete. Be sure to request a higher sending limit at the same time. + +#### Request new phone numbers + +1. Go to Pinpoint console for the same region you are using SNS in. +1. In the lefthand sidebar, go the `SMS and Voice` (bottom) and choose `Phone Numbers` +1. Under `Number Settings` choose `Request Phone Number` +1. Choose Toll-free number, tick SMS, untick Voice, choose `transactional`, hit next and then `request` +1. Select `Toll-free registrations` and `Create registration` +1. Select the number you just created and then `Register existing toll-free number` +1. Complete and submit the form. Approval usually takes about 2 weeks. +1. Set this phone number as the `AWS_US_TOLL_FREE_NUMBER` in the environment you are creating diff --git a/sample.env b/sample.env index 464b17cf1..3748ae331 100644 --- a/sample.env +++ b/sample.env @@ -1,14 +1,14 @@ # STEPS TO SET UP # -# 1. Pull down AWS creds from cloud.gov using `cf env`, then update AWS section +# 1. Copy this file to `.env` # -# 2. Update NEW_RELIC_LICENSE_KEY +# 2. Update NEW_RELIC_LICENSE_KEY, if you are testing new relic integrations in development # # 3. Uncomment either the Docker setup or the direct setup # # 4. Comment out the other setup # -# 5. If needed, set `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) +# 5. Run `cd terraform/development; ./run.sh` to include service credentials in `.env` # # ## REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### @@ -16,26 +16,23 @@ ############################################################# # AWS -AWS_ACCESS_KEY_ID="don't write secrets to the sample file" -AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" -AWS_REGION=us-west-2 -AWS_US_TOLL_FREE_NUMBER=+18446120782 +AWS_US_TOLL_FREE_NUMBER=+18556438890 ############################################################# # Local Docker setup, all overwritten in cloud.gov -ADMIN_BASE_URL=http://admin:6012 -API_HOST_NAME=http://dev:6011 -REDIS_URL=redis://redis:6380 -DATABASE_URL=postgresql://postgres:chummy@db:5432/notification_api -SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api +# ADMIN_BASE_URL=http://admin:6012 +# API_HOST_NAME=http://dev:6011 +# REDIS_URL=redis://redis:6380 +# DATABASE_URL=postgresql://postgres:chummy@db:5432/notification_api +# SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api # Local direct setup, all overwritten in cloud.gov -# ADMIN_BASE_URL=http://localhost:6012 -# API_HOST_NAME=http://localhost:6011 -# REDIS_URL=redis://localhost:6379 -# DATABASE_URL=postgresql://localhost:5432/notification_api -# SQLALCHEMY_DATABASE_TEST_URI=postgresql://localhost:5432/test_notification_api +ADMIN_BASE_URL=http://localhost:6012 +API_HOST_NAME=http://localhost:6011 +REDIS_URL=redis://localhost:6379 +DATABASE_URL=postgresql://localhost:5432/notification_api +SQLALCHEMY_DATABASE_TEST_URI=postgresql://localhost:5432/test_notification_api ############################################################# @@ -47,7 +44,6 @@ NOTIFY_ENVIRONMENT=development STATSD_HOST=localhost SES_STUB_URL=None NOTIFY_APP_NAME=api -# NOTIFY_EMAIL_DOMAIN=notify.sandbox.10x.gsa.gov ############################################################# diff --git a/terraform/development/main.tf b/terraform/development/main.tf new file mode 100644 index 000000000..75c9bfe37 --- /dev/null +++ b/terraform/development/main.tf @@ -0,0 +1,91 @@ +locals { + cf_org_name = "gsa-tts-benefits-studio-prototyping" + cf_space_name = "notify-local-dev" + recursive_delete = true + key_name = "${var.username}-dev-key" +} + +module "csv_upload_bucket" { + source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + recursive_delete = local.recursive_delete + name = "${var.username}-csv-upload-bucket" +} +resource "cloudfoundry_service_key" "csv_key" { + name = local.key_name + service_instance = module.csv_upload_bucket.bucket_id +} + +module "contact_list_bucket" { + source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0" + + cf_org_name = local.cf_org_name + cf_space_name = local.cf_space_name + recursive_delete = local.recursive_delete + name = "${var.username}-contact-list-bucket" +} +resource "cloudfoundry_service_key" "contact_list_key" { + name = local.key_name + service_instance = module.contact_list_bucket.bucket_id +} + +data "cloudfoundry_space" "staging" { + org_name = local.cf_org_name + name = "notify-staging" +} + +data "cloudfoundry_service_instance" "ses_email" { + name_or_id = "notify-api-ses-staging" + space = data.cloudfoundry_space.staging.id +} +resource "cloudfoundry_service_key" "ses_key" { + name = local.key_name + service_instance = data.cloudfoundry_service_instance.ses_email.id +} + +data "cloudfoundry_service_instance" "sns_sms" { + name_or_id = "notify-api-sns-staging" + space = data.cloudfoundry_space.staging.id +} +resource "cloudfoundry_service_key" "sns_key" { + name = local.key_name + service_instance = data.cloudfoundry_service_instance.sns_sms.id +} + +locals { + credentials = <> .env" + } +} diff --git a/terraform/development/providers.tf b/terraform/development/providers.tf new file mode 100644 index 000000000..59bb98a70 --- /dev/null +++ b/terraform/development/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_version = "~> 1.0" + required_providers { + cloudfoundry = { + source = "cloudfoundry-community/cloudfoundry" + version = "0.50.5" + } + } +} + +provider "cloudfoundry" { + api_url = "https://api.fr.cloud.gov" + user = var.cf_user + password = var.cf_password + app_logs_max = 30 +} diff --git a/terraform/development/run.sh b/terraform/development/run.sh new file mode 100755 index 000000000..5b1fbf7c2 --- /dev/null +++ b/terraform/development/run.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +username=`whoami` +org="gsa-tts-benefits-studio-prototyping" + +usage=" +$0: Create development infrastructure + +Usage: + $0 -h + $0 [-u ] [-k] + +Options: +-h: show help and exit +-u : your username. Default: $username +-k: keep service user. Default is to remove them after run +-d: Destroy development resources. Default is to create them + +Notes: +* Requires cf-cli@8 +" + +action="apply" +creds="remove" + +while getopts ":hkdu:" opt; do + case "$opt" in + u) + username=${OPTARG} + ;; + k) + creds="keep" + ;; + d) + action="destroy" + ;; + h) + echo "$usage" + exit 0 + ;; + esac +done + +set -e + +service_account="$username-terraform" + +if [[ ! -f "secrets.auto.tfvars" ]]; then + # create user in notify-local-dev space to create s3 buckets + ../create_service_account.sh -s notify-local-dev -u $service_account > secrets.auto.tfvars + + # grant user access to notify-staging to create a service key for SES and SNS + cg_username=`cf service-key $service_account service-account-key | tail -n +2 | jq -r '.credentials.username'` + cf set-space-role $cg_username $org notify-staging SpaceDeveloper +fi + +set +e + +terraform init +terraform $action -var="username=$username" + +set -e + +if [[ $creds = "remove" ]]; then + ../destroy_service_account.sh -s notify-local-dev -u $service_account + rm secrets.auto.tfvars +fi + +exit 0 diff --git a/terraform/development/variables.tf b/terraform/development/variables.tf new file mode 100644 index 000000000..2f5137920 --- /dev/null +++ b/terraform/development/variables.tf @@ -0,0 +1,5 @@ +variable "cf_password" { + sensitive = true +} +variable "cf_user" {} +variable "username" {} From 84e7e9b5cf33dfa79ab9f7fda474132f4ed50a10 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 13 Mar 2023 12:59:44 -0400 Subject: [PATCH 2/5] Use credentials output by terraform/development --- .github/workflows/checks.yml | 3 +-- .github/workflows/daily_checks.yml | 4 +--- app/cloudfoundry_config.py | 17 +++++++++-------- app/config.py | 18 +++++++----------- tests/app/aws/test_s3.py | 6 +++--- tests/app/clients/test_aws_sns.py | 2 +- tests/app/delivery/test_send_to_providers.py | 3 ++- 7 files changed, 24 insertions(+), 29 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8347f3b77..95d343cb6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -13,8 +13,7 @@ env: FLASK_APP: application.py WERKZEUG_DEBUG_PIN: off REDIS_ENABLED: 0 - AWS_REGION: us-west-2 - AWS_US_TOLL_FREE_NUMBER: "+18446120782" + AWS_US_TOLL_FREE_NUMBER: "+18556438890" jobs: build: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index b3d113b20..dfd0e83b4 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -16,10 +16,8 @@ env: NEW_RELIC_ENVIRONMENT: test FLASK_APP: application.py WERKZEUG_DEBUG_PIN: off - NOTIFY_EMAIL_DOMAIN: dispostable.com REDIS_ENABLED: 0 - AWS_REGION: us-west-2 - AWS_US_TOLL_FREE_NUMBER: "+18446120782" + AWS_US_TOLL_FREE_NUMBER: "+18556438890" jobs: pip-audit: diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index a6c3ad075..7fda0184d 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -34,51 +34,52 @@ class CloudfoundryConfig: @property def ses_email_domain(self): try: - return self._ses_credentials('domain_arn').split('/')[-1] + domain_arn = self._ses_credentials('domain_arn') except KeyError: - return getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') + domain_arn = getenv('SES_DOMAIN_ARN', 'dev.notify.gov') + return domain_arn.split('/')[-1] @property def ses_region(self): try: return self._ses_credentials('region') except KeyError: - return getenv('AWS_REGION') + return getenv('SES_AWS_REGION', 'us-west-1') @property def ses_access_key(self): try: return self._ses_credentials('smtp_user') except KeyError: - return getenv('AWS_ACCESS_KEY_ID') + return getenv('SES_AWS_ACCESS_KEY_ID') @property def ses_secret_key(self): try: return self._ses_credentials('secret_access_key') except KeyError: - return getenv('AWS_SECRET_ACCESS_KEY') + return getenv('SES_AWS_SECRET_ACCESS_KEY') @property def sns_access_key(self): try: return self._sns_credentials('aws_access_key_id') except KeyError: - return getenv('AWS_ACCESS_KEY_ID') + return getenv('SNS_AWS_ACCESS_KEY_ID') @property def sns_secret_key(self): try: return self._sns_credentials('aws_secret_access_key') except KeyError: - return getenv('AWS_SECRET_ACCESS_KEY') + return getenv('SNS_AWS_SECRET_ACCESS_KEY') @property def sns_region(self): try: return self._sns_credentials('region') except KeyError: - return getenv('AWS_REGION') + return getenv('SNS_AWS_REGION', 'us-west-1') @property def sns_topic_arns(self): diff --git a/app/config.py b/app/config.py index 2b09617b3..ce38b9223 100644 --- a/app/config.py +++ b/app/config.py @@ -93,7 +93,6 @@ class Config(object): EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 # AWS Settings - AWS_REGION = getenv('AWS_REGION') AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER") # Whether to ignore POSTs from SNS for replies to SMS we sent RECEIVE_INBOUND_SMS = False @@ -279,12 +278,12 @@ class Config(object): DOCUMENT_DOWNLOAD_API_KEY = getenv('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') -def _default_s3_credentials(bucket_name): +def _s3_credentials_from_env(bucket_prefix): return { - 'bucket': bucket_name, - 'access_key_id': getenv('AWS_ACCESS_KEY_ID'), - 'secret_access_key': getenv('AWS_SECRET_ACCESS_KEY'), - 'region': getenv('AWS_REGION') + 'bucket': getenv(f"{bucket_prefix}_BUCKET_NAME"), + 'access_key_id': getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"), + 'secret_access_key': getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"), + 'region': getenv(f"{bucket_prefix}_AWS_REGION") } @@ -294,8 +293,8 @@ class Development(Config): DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] # Buckets - CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') - CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') + CSV_UPLOAD_BUCKET = _s3_credentials_from_env('CSV') + CONTACT_LIST_BUCKET = _s3_credentials_from_env('CONTACT') # credential overrides DANGEROUS_SALT = 'development-notify-salt' @@ -317,9 +316,6 @@ class Test(Development): '10d1b9c9-0072-4fa9-ae1c-595e333841da', ] - CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload') - CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') - # this is overriden in CI SQLALCHEMY_DATABASE_URI = getenv('SQLALCHEMY_DATABASE_TEST_URI') diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index 735a7a141..b339c94a4 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -3,9 +3,9 @@ from os import getenv from app.aws.s3 import get_s3_file -default_access_key = getenv('AWS_ACCESS_KEY_ID') -default_secret_key = getenv('AWS_SECRET_ACCESS_KEY') -default_region = getenv('AWS_REGION') +default_access_key = getenv('CSV_AWS_ACCESS_KEY_ID') +default_secret_key = getenv('CSV_AWS_SECRET_ACCESS_KEY') +default_region = getenv('CSV_AWS_REGION') def single_s3_object_stub(key='foo', last_modified=None): diff --git a/tests/app/clients/test_aws_sns.py b/tests/app/clients/test_aws_sns.py index 0a3db94c2..514e19d84 100644 --- a/tests/app/clients/test_aws_sns.py +++ b/tests/app/clients/test_aws_sns.py @@ -15,7 +15,7 @@ def test_send_sms_successful_returns_aws_sns_response(notify_api, mocker): Message=content, MessageAttributes={ 'AWS.SNS.SMS.SMSType': {'DataType': 'String', 'StringValue': 'Transactional'}, - 'AWS.MM.SMS.OriginationNumber': {'DataType': 'String', 'StringValue': '+18446120782'} + 'AWS.MM.SMS.OriginationNumber': {'DataType': 'String', 'StringValue': '+18556438890'} } ) diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 48ff35368..c5bd57648 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -11,6 +11,7 @@ from requests import HTTPError import app from app import aws_sns_client, notification_provider_clients +from app.cloudfoundry_config import cloud_config from app.dao import notifications_dao from app.dao.provider_details_dao import get_provider_details_by_identifier from app.delivery import send_to_providers @@ -164,7 +165,7 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist ) app.aws_ses_client.send_email.assert_called_once_with( - '"Sample service" ', + f"\"Sample service\" ", 'jo.smith@example.com', 'Jo some HTML', body='Hello Jo\nThis is an email from GOV.\u200bUK with some HTML\n', From 778a41a1cba95f4befdc2c51fc2fe8f469792d5f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 13 Mar 2023 14:15:22 -0400 Subject: [PATCH 3/5] Put api in key name so we can easily add a key for shared buckets to admin --- terraform/development/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/development/main.tf b/terraform/development/main.tf index 75c9bfe37..c243f0376 100644 --- a/terraform/development/main.tf +++ b/terraform/development/main.tf @@ -2,7 +2,7 @@ locals { cf_org_name = "gsa-tts-benefits-studio-prototyping" cf_space_name = "notify-local-dev" recursive_delete = true - key_name = "${var.username}-dev-key" + key_name = "${var.username}-api-dev-key" } module "csv_upload_bucket" { From 9eb075f074e11f2cc76f1b2f3ea5654447cf9a43 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 14 Mar 2023 15:36:32 -0400 Subject: [PATCH 4/5] Address PR feedback --- README.md | 32 +++++++++++++++++------------ sample.env | 13 +----------- terraform/create_service_account.sh | 15 +++++++------- terraform/development/run.sh | 6 +++++- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 6161d442e..34e367374 100644 --- a/README.md +++ b/README.md @@ -42,19 +42,30 @@ Our other repositories are: ## Local setup +### Common steps + +1. Install pre-requisites for setup: + * [jq](https://stedolan.github.io/jq/): `brew install jq` + * [terraform](https://www.terraform.io/): `brew install terraform` or `brew install tfenv` and use `tfenv` to install `terraform ~> 1.4.0` + * [cf-cli@8](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html): `brew install cloudfoundry/tap/cf-cli@8` +1. [Log into cloud.gov](https://cloud.gov/docs/getting-started/setup/#set-up-the-command-line): `cf login -a api.fr.cloud.gov --sso` +1. Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov +1. Run the development terraform with: + + ``` + $ cd terraform/development + $ ./run.sh + ``` + +1. If you want to send data to New Relic from your local develpment environment, set `NEW_RELIC_LICENSE_KEY` within `.env` +1. Follow the instructions for either `Direct installation` or `Docker installation` below + ### Direct installation 1. Set up Postgres && Redis on your machine 1. Install [pipenv](https://pipenv.pypa.io/en/latest/) -1. Create the .env file - - ``` - cp sample.env .env - # follow the instructions in .env - ``` - 1. Run the project setup `make bootstrap` @@ -78,12 +89,7 @@ Our other repositories are: If you're working in VS Code, you can also leverage Docker for a containerized dev environment -1. Create the .env file - - ``` - cp sample.env .env - # follow the instructions in .env - ``` +1. Uncomment the `Local Docker setup` lines in `.env` and comment out the `Local direct setup` lines. 1. Install the Remote-Containers plug-in in VS Code diff --git a/sample.env b/sample.env index 3748ae331..71a451e1d 100644 --- a/sample.env +++ b/sample.env @@ -1,15 +1,4 @@ -# STEPS TO SET UP -# -# 1. Copy this file to `.env` -# -# 2. Update NEW_RELIC_LICENSE_KEY, if you are testing new relic integrations in development -# -# 3. Uncomment either the Docker setup or the direct setup -# -# 4. Comment out the other setup -# -# 5. Run `cd terraform/development; ./run.sh` to include service credentials in `.env` -# +# See README.md for local setup instructions # ## REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### diff --git a/terraform/create_service_account.sh b/terraform/create_service_account.sh index cee07591e..b4c44e705 100755 --- a/terraform/create_service_account.sh +++ b/terraform/create_service_account.sh @@ -19,14 +19,15 @@ Options: Notes: * OrgManager is required for terraform to create -egress spaces -* Requires cf-cli@8 +* Requires cf-cli@8 & jq " cf_version=`cf --version | cut -d " " -f 3` if [[ $cf_version != 8.* ]]; then - echo "$usage" + echo "$usage" >&2 exit 1 fi +command -v jq >/dev/null || { echo "$usage" >&2; exit 1; } set -e set -o pipefail @@ -61,17 +62,17 @@ while getopts ":hms:u:r:o:" opt; do done if [[ $space = "" || $service = "" ]]; then - echo "$usage" + echo "$usage" >&2 exit 1 fi -cf target -o $org -s $space 1>&2 +cf target -o $org -s $space >&2 # create user account service -cf create-service cloud-gov-service-account $role $service 1>&2 +cf create-service cloud-gov-service-account $role $service >&2 # create service key -cf create-service-key $service service-account-key 1>&2 +cf create-service-key $service service-account-key >&2 # output service key to stdout in secrets.auto.tfvars format creds=`cf service-key $service service-account-key | tail -n +2 | jq '.credentials'` @@ -79,7 +80,7 @@ username=`echo $creds | jq -r '.username'` password=`echo $creds | jq -r '.password'` if [[ $org_manager = "true" ]]; then - cf set-org-role $username $org OrgManager 1>&2 + cf set-org-role $username $org OrgManager >&2 fi cat << EOF diff --git a/terraform/development/run.sh b/terraform/development/run.sh index 5b1fbf7c2..f9b6c6838 100755 --- a/terraform/development/run.sh +++ b/terraform/development/run.sh @@ -45,7 +45,7 @@ set -e service_account="$username-terraform" -if [[ ! -f "secrets.auto.tfvars" ]]; then +if [[ ! -s "secrets.auto.tfvars" ]]; then # create user in notify-local-dev space to create s3 buckets ../create_service_account.sh -s notify-local-dev -u $service_account > secrets.auto.tfvars @@ -54,6 +54,10 @@ if [[ ! -f "secrets.auto.tfvars" ]]; then cf set-space-role $cg_username $org notify-staging SpaceDeveloper fi +if [[ ! -f "../../.env" ]]; then + cp ../../sample.env ../../.env +fi + set +e terraform init From bea7a55e4a2d4764e12cb71fe8a5d5f9a319f729 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Wed, 15 Mar 2023 16:24:02 -0400 Subject: [PATCH 5/5] Make sure we're in the correct directory this will make relative paths and terraform find the correct files --- terraform/development/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/development/run.sh b/terraform/development/run.sh index f9b6c6838..0285d0946 100755 --- a/terraform/development/run.sh +++ b/terraform/development/run.sh @@ -43,6 +43,9 @@ done set -e +# ensure we're in the correct directory +cd $(dirname $0) + service_account="$username-terraform" if [[ ! -s "secrets.auto.tfvars" ]]; then