darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 02:23:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into jim/091422/deliverycallbacks

Browse Source
This commit is contained in:
Jim Moffet
2022-09-30 11:21:46 -04:00
committed by GitHub
parent 48af6f7c23 36ce2f7830
commit d0bba8a8bd
44 changed files with 917 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/checks.yml
+3 -2
View File
@@ -9,7 +9,6 @@ env:
DEBUG: True
ANTIVIRUS_ENABLED: 0
NOTIFY_ENVIRONMENT: test
NOTIFICATION_QUEUE_PREFIX: local_dev_10x
STATSD_HOST: localhost
SES_STUB_URL: None
NOTIFY_APP_NAME: api
@@ -28,6 +27,8 @@ env:
AWS_REGION: us-west-2
AWS_PINPOINT_REGION: us-west-2
AWS_US_TOLL_FREE_NUMBER: +18446120782
AWS_ACCESS_KEY_ID: not-a-real-key-id
AWS_SECRET_ACCESS_KEY: not-a-real-secret
jobs:
build:
@@ -116,7 +117,7 @@ jobs:
- name: Run OWASP Baseline Scan
uses: zaproxy/action-api-scan@v0.1.1
with:
docker_name: 'owasp/zap2docker-weekly'
docker_name: 'owasp/zap2docker-stable'
target: 'http://localhost:6011/_status'
fail_action: true
allow_issue_writing: false
.github/workflows/daily_checks.yml
-1
View File
@@ -13,7 +13,6 @@ env:
DEBUG: True
ANTIVIRUS_ENABLED: 0
NOTIFY_ENVIRONMENT: test
NOTIFICATION_QUEUE_PREFIX: local_dev_10x
STATSD_HOST: localhost
SES_STUB_URL: None
NOTIFY_APP_NAME: api
.github/workflows/deploy.yml
+25
View File
@@ -23,6 +23,30 @@ jobs:
libcurl4-openssl-dev
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Check for changes to Terraform
id: changed-terraform-files
uses: tj-actions/changed-files@v1.1.2
with:
files: terraform/staging
- name: Terraform init
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
run: terraform init
- name: Terraform apply
if: steps.changed-terraform-files.outputs.any_changed == 'true'
working-directory: terraform/staging
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
run: terraform apply -auto-approve -input=false
- name: Set up Python 3.9
uses: actions/setup-python@v3
@@ -46,6 +70,7 @@ jobs:
cf_org: gsa-10x-prototyping
cf_space: 10x-notifications
push_arguments: >-
--var env=staging
--var DANGEROUS_SALT="$DANGEROUS_SALT"
--var SECRET_KEY="$SECRET_KEY"
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
.github/workflows/terraform-production.yml
+79
View File
@@ -0,0 +1,79 @@
name: Run Terraform plan in production
on:
pull_request:
branches: [ production ]
paths: [ 'terraform/**' ]
defaults:
run:
working-directory: terraform/production
jobs:
terraform:
name: Terraform plan
runs-on: ubuntu-latest
environment: production
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Terraform format
id: format
run: terraform fmt -check
- name: Terraform init
id: init
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
run: terraform init
- name: Terraform validate
id: validation
run: terraform validate -no-color
- name: Terraform plan
id: plan
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
- name: Read Terraform plan output file
id: terraform_output
uses: juliangruber/read-file-action@v1
if: ${{ always() }}
with:
path: ./terraform/production/plan_output.txt
# inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
- name: Update PR
uses: actions/github-script@v4
# we would like to update the PR even when a prior step failed
if: ${{ always() }}
with:
script: |
const output = `Terraform Format and Style: ${{ steps.format.outcome }}
Terraform Initialization: ${{ steps.init.outcome }}
Terraform Validation: ${{ steps.validation.outcome }}
Terraform Plan: ${{ steps.plan.outcome }}
<details><summary>Show Plan</summary>
\`\`\`\n
${{ steps.terraform_output.outputs.content }}
\`\`\`
</details>
*Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
github.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: output
})
.github/workflows/terraform-staging.yml
+78
View File
@@ -0,0 +1,78 @@
name: Run Terraform plan in staging
on:
pull_request:
branches: [ main ]
paths: [ 'terraform/**' ]
defaults:
run:
working-directory: terraform/staging
jobs:
terraform:
name: Terraform plan
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Terraform format
id: format
run: terraform fmt -check
- name: Terraform init
id: init
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
run: terraform init
- name: Terraform validate
id: validation
run: terraform validate -no-color
- name: Terraform plan
id: plan
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
- name: Read Terraform plan output file
id: terraform_output
uses: juliangruber/read-file-action@v1
if: ${{ always() }}
with:
path: ./terraform/staging/plan_output.txt
# inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
- name: Update PR
uses: actions/github-script@v4
# we would like to update the PR even when a prior step failed
if: ${{ always() }}
with:
script: |
const output = `Terraform Format and Style: ${{ steps.format.outcome }}
Terraform Initialization: ${{ steps.init.outcome }}
Terraform Validation: ${{ steps.validation.outcome }}
Terraform Plan: ${{ steps.plan.outcome }}
<details><summary>Show Plan</summary>
\`\`\`\n
${{ steps.terraform_output.outputs.content }}
\`\`\`
</details>
*Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
github.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: output
})
.gitignore
+7
View File
@@ -82,3 +82,10 @@ varsfile*
.secret*
/scripts/run_my_tests.sh
# Terraform
.terraform.lock.hcl
**/.terraform/*
secrets.auto.tfvars
terraform.tfstate
terraform.tfstate.backup
README.md
-1
View File
@@ -49,7 +49,6 @@ NOTE: when you change .env in the future, you'll need to rebuild the devcontaine
Things to change:
- If you're not the first to deploy, only replace the aws creds, get these from team lead
- Replace `NOTIFICATION_QUEUE_PREFIX` with `local_dev_<your org>_`
- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project)
- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values
- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars
app/aws/s3.py
+31 -17
View File
@@ -1,30 +1,30 @@
import os
import botocore
from boto3 import client, resource
from boto3 import Session, client
from flask import current_app
FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
default_access_key = os.environ.get('AWS_ACCESS_KEY_ID')
default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
default_region = os.environ.get('AWS_REGION')
def get_s3_file(bucket_name, file_location):
s3_file = get_s3_object(bucket_name, file_location)
def get_s3_file(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
return s3_file.get()['Body'].read().decode('utf-8')
def get_s3_object(bucket_name, file_location):
s3 = resource('s3')
def get_s3_object(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
s3 = session.resource('s3')
return s3.Object(bucket_name, file_location)
def head_s3_object(bucket_name, file_location):
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
boto_client = client('s3', current_app.config['AWS_REGION'])
return boto_client.head_object(Bucket=bucket_name, Key=file_location)
def file_exists(bucket_name, file_location):
def file_exists(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
try:
# try and access metadata of object
get_s3_object(bucket_name, file_location).metadata
get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata
return True
except botocore.exceptions.ClientError as e:
if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
@@ -36,6 +36,9 @@ def get_job_location(service_id, job_id):
return (
current_app.config['CSV_UPLOAD_BUCKET_NAME'],
FILE_LOCATION_STRUCTURE.format(service_id, job_id),
current_app.config['CSV_UPLOAD_ACCESS_KEY'],
current_app.config['CSV_UPLOAD_SECRET_KEY'],
current_app.config['CSV_UPLOAD_REGION'],
)
@@ -43,6 +46,9 @@ def get_contact_list_location(service_id, contact_list_id):
return (
current_app.config['CONTACT_LIST_BUCKET_NAME'],
FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
current_app.config['CONTACT_LIST_ACCESS_KEY'],
current_app.config['CONTACT_LIST_SECRET_KEY'],
current_app.config['CONTACT_LIST_REGION'],
)
@@ -69,13 +75,21 @@ def remove_contact_list_from_s3(service_id, contact_list_id):
return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))
def remove_s3_object(bucket_name, object_key):
obj = get_s3_object(bucket_name, object_key)
def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
return obj.delete()
def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None):
s3_client = client('s3', current_app.config['AWS_REGION'])
def get_list_of_files_by_suffix(
bucket_name,
subfolder='',
suffix='',
last_modified=None,
access_key=default_access_key,
secret_key=default_secret_key,
region=default_region
):
s3_client = client('s3', region, aws_access_key_id=access_key, aws_secret_access_key=secret_key)
paginator = s3_client.get_paginator('list_objects_v2')
page_iterator = paginator.paginate(
app/celery/tasks.py
+1
View File
@@ -449,6 +449,7 @@ def handle_exception(task, notification, notification_id, exc):
# Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
# SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
# send to the retry queue.
# This probably (hopefully) is not an issue with Redis as the celery backing store
current_app.logger.exception('Retry' + retry_msg)
try:
task.retry(queue=QueueNames.RETRY, exc=exc)
app/cloudfoundry_config.py
+22
View File
@@ -2,6 +2,12 @@ import json
import os
def find_by_service_name(services, service_name):
for i in range(len(services)):
if services[i]['name'] == service_name:
return services[i]
return None
def extract_cloudfoundry_config():
vcap_services = json.loads(os.environ['VCAP_SERVICES'])
@@ -9,3 +15,19 @@ def extract_cloudfoundry_config():
os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
# Redis config
os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')
# CSV Upload Bucket Name
bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
# Contact List Bucket Name
bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']
app/config.py
+56 -51
View File
@@ -114,9 +114,6 @@ class Config(object):
FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
# Prefix to identify queues in SQS
NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX')
# Use notify.sandbox.10x sending domain unless overwritten by environment
NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
@@ -203,11 +200,9 @@ class Config(object):
DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
CELERY = {
'broker_url': 'sqs://',
'broker_url': REDIS_URL,
'broker_transport_options': {
'region': AWS_REGION,
'visibility_timeout': 310,
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
},
'timezone': 'Europe/London',
'imports': [
@@ -418,14 +413,20 @@ class Development(Config):
REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2')
CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2')
# TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
# DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
# LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
# LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
# INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
# TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
# LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
# INTERNAL_CLIENT_API_KEYS = {
# Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
@@ -444,7 +445,6 @@ class Development(Config):
NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
REDIS_URL = os.environ.get('REDIS_URL')
ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
@@ -473,13 +473,13 @@ class Test(Development):
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
# TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
# DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
# LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
# LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
# INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
# TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
# LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
# this is overriden in CI
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api')
@@ -510,13 +510,13 @@ class Preview(Config):
NOTIFY_ENVIRONMENT = 'preview'
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
CONTACT_LIST_BUCKET_NAME = 'preview-contact-list'
TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
# TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
# DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
# LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
# LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
# INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
# TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
# LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
FROM_NUMBER = 'preview'
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = False
@@ -527,13 +527,13 @@ class Staging(Config):
NOTIFY_ENVIRONMENT = 'staging'
CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
CONTACT_LIST_BUCKET_NAME = 'staging-contact-list'
TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
# TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
# DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
# LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
# LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
# INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
# TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
# LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
FROM_NUMBER = 'stage'
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = True
@@ -542,16 +542,22 @@ class Staging(Config):
class Live(Config):
NOTIFY_ENVIRONMENT = 'live'
# buckets
CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox
CONTACT_LIST_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox
CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox
CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox
CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
# TODO: verify below buckets only used for letters
TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox
DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox
LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox
LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox
INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
# TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox
# DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox
# LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox
# LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox
# INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
# TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
# LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
FROM_NUMBER = 'US Notify'
API_RATE_LIMIT_ENABLED = True
@@ -563,7 +569,6 @@ class Live(Config):
REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log')
REDIS_URL = os.environ.get('REDIS_URL')
class CloudFoundryConfig(Config):
@@ -576,12 +581,12 @@ class Sandbox(CloudFoundryConfig):
NOTIFY_ENVIRONMENT = 'sandbox'
CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list'
LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
# LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
# TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
# DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
# LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
# LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
# INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
FROM_NUMBER = 'sandbox'
app/letters/rest.py
+1
View File
@@ -10,6 +10,7 @@ letter_job = Blueprint("letter-job", __name__)
register_errors(letter_job)
# too many references will make SQS error (as the task can only be 256kb)
# Maybe doesn't matter anymore with Redis as the celery backing store
MAX_REFERENCES_PER_TASK = 5000
app/v2/notifications/post_notifications.py
+1
View File
@@ -236,6 +236,7 @@ def process_sms_or_email_notification(
# If SQS cannot put the task on the queue, it's probably because the notification body was too long and it
# went over SQS's 256kb message limit. If the body is very large, it may exceed the HTTP max content length;
# the exception we get here isn't handled correctly by botocore - we get a ResponseParserError instead.
# Hopefully this is no longer an issue with Redis as celery's backing store
current_app.logger.info(
f'Notification {notification_id} failed to save to high volume queue. Using normal flow instead'
)
pytest.ini
-1
View File
@@ -4,7 +4,6 @@ env =
NOTIFY_ENVIRONMENT=test
MMG_API_KEY=mmg-secret-key
FIRETEXT_API_KEY=Firetext
NOTIFICATION_QUEUE_PREFIX=testing
REDIS_ENABLED=0
addopts = -p no:warnings
xfail_strict = true
requirements.in
+1 -1
View File
@@ -2,7 +2,7 @@
# with package version changes made in requirements.in
cffi==1.15.0
celery[sqs]==5.2.6
celery[redis]==5.2.7
Flask-Bcrypt==1.0.1
flask-marshmallow==0.14.0
Flask-Migrate==3.1.0
requirements.txt
+5 -3
View File
@@ -41,7 +41,7 @@ cachetools==5.1.0
# via
# -r requirements.in
# notifications-utils
celery[sqs]==5.2.6
celery[redis]==5.2.7
# via -r requirements.in
certifi==2022.5.18.1
# via
@@ -150,7 +150,7 @@ kombu==5.2.4
# via celery
lxml==4.9.1
# via -r requirements.in
mako==1.2.0
mako==1.2.2
# via alembic
markupsafe==2.1.1
# via
@@ -223,7 +223,9 @@ pyyaml==5.4.1
# awscli
# notifications-utils
redis==4.3.1
# via flask-redis
# via
# celery
# flask-redis
requests==2.27.1
# via
# awscli-cwlogs
sample.env
-1
View File
@@ -4,7 +4,6 @@
DEBUG=True
ANTIVIRUS_ENABLED=0
NOTIFY_ENVIRONMENT=development
NOTIFICATION_QUEUE_PREFIX=local_dev_YOURNAME_
STATSD_HOST=localhost
SES_STUB_URL=None
NOTIFY_APP_NAME=api
terraform/README.md
+130
View File
@@ -0,0 +1,130 @@
# Terraform
This directory holds the terraform modules for maintaining your complete persistent infrastructure.
Prerequisite: install the `jq` JSON processor: `brew install jq`
## Initial setup
1. Manually run the bootstrap module following instructions under `Terraform State Credentials`
1. Setup CI/CD Pipeline to run Terraform
1. Copy bootstrap credentials to your CI/CD secrets using the instructions in the base README
1. Create a cloud.gov SpaceDeployer by following the instructions under `SpaceDeployers`
1. Copy SpaceDeployer credentials to your CI/CD secrets using the instructions in the base README
1. Manually Running Terraform
1. Follow instructions under `Set up a new environment` to create your infrastructure
## Terraform State Credentials
The bootstrap module is used to create an s3 bucket for later terraform runs to store their state in.
### Bootstrapping the state storage s3 buckets for the first time
1. Run `terraform init`
1. Run `./run.sh plan` to verify that the changes are what you expect
1. Run `./run.sh apply` to set up the bucket and retrieve credentials
1. Follow instructions under `Use bootstrap credentials`
1. Ensure that `import.sh` includes a line and correct IDs for any resources created
1. Run `./teardown_creds.sh` to remove the space deployer account used to create the s3 bucket
### To make changes to the bootstrap module
*This should not be necessary in most cases*
1. Run `terraform init`
1. If you don't have terraform state locally:
1. run `./import.sh`
1. optionally run `./run.sh apply` to include the existing outputs in the state file
1. Make your changes
1. Continue from step 2 of the boostrapping instructions
### Retrieving existing bucket credentials
1. Run `./run.sh show`
1. Follow instructions under `Use bootstrap credentials`
#### Use bootstrap credentials
1. Add the following to `~/.aws/credentials`
```
[notify-terraform-backend]
aws_access_key_id = <access_key_id from bucket_credentials>
aws_secret_access_key = <secret_access_key from bucket_credentials>
```
1. Copy `bucket` from `bucket_credentials` output to the backend block of `staging/providers.tf` and `production/providers.tf`
## SpaceDeployers
A [SpaceDeployer](https://cloud.gov/docs/services/cloud-gov-service-account/) account is required to run terraform or
deploy the application from the CI/CD pipeline. Create a new account by running:
`./create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME>`
## Set up a new environment manually
The below steps rely on you first configuring access to the Terraform state in s3 as described in [Terraform State Credentials](#terraform-state-credentials).
1. `cd` to the environment you are working in
1. Set up a SpaceDeployer
```bash
# create a space deployer service instance that can log in with just a username and password
# the value of < SPACE_NAME > should be `staging` or `prod` depending on where you are working
# the value for < ACCOUNT_NAME > can be anything, although we recommend
# something that communicates the purpose of the deployer
# for example: circleci-deployer for the credentials CircleCI uses to
# deploy the application or <your_name>-terraform for credentials to run terraform manually
./create_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME> > secrets.auto.tfvars
```
The script will output the `username` (as `cf_user`) and `password` (as `cf_password`) for your `<ACCOUNT_NAME>`. Read more in the [cloud.gov service account documentation](https://cloud.gov/docs/services/cloud-gov-service-account/).
The easiest way to use this script is to redirect the output directly to the `secrets.auto.tfvars` file it needs to be used in
1. Run terraform from your new environment directory with
```bash
terraform init
terraform plan
```
1. Apply changes with `terraform apply`.
1. Remove the space deployer service instance if it doesn't need to be used again, such as when manually running terraform once.
```bash
# <SPACE_NAME> and <ACCOUNT_NAME> have the same values as used above.
./destroy_service_account.sh -s <SPACE_NAME> -u <ACCOUNT_NAME>
```
## Structure
Each environment has its own module, which relies on a shared module for everything except the providers code and environment specific variables and settings.
```
- bootstrap/
|- main.tf
|- providers.tf
|- variables.tf
|- run.sh
|- teardown_creds.sh
|- import.sh
- <env>/
|- main.tf
|- providers.tf
|- secrets.auto.tfvars
|- variables.tf
```
In the environment-specific modules:
- `providers.tf` lists the required providers
- `main.tf` calls the shared Terraform code, but this is also a place where you can add any other services, resources, etc, which you would like to set up for that environment
- `variables.tf` lists the variables that will be needed, either to pass through to the child module or for use in this module
- `secrets.auto.tfvars` is a file which contains the information about the service-key and other secrets that should not be shared
In the bootstrap module:
- `providers.tf` lists the required providers
- `main.tf` sets up s3 bucket to be shared across all environments. It lives in `prod` to communicate that it should not be deleted
- `variables.tf` lists the variables that will be needed. Most values are hard-coded in this module
- `run.sh` Helper script to set up a space deployer and run terraform. The terraform action (`show`/`plan`/`apply`/`destroy`) is passed as an argument
- `teardown_creds.sh` Helper script to remove the space deployer setup as part of `run.sh`
- `import.sh` Helper script to create a new local state file in case terraform changes are needed
terraform/bootstrap/import.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
read -p "Are you sure you want to import terraform state (y/n)? " verify
if [[ $verify == "y" ]]; then
echo "Importing bootstrap state"
./run.sh import module.s3.cloudfoundry_service_instance.bucket 31204bcc-aae3-4cd3-8b59-5055a338d44f
./run.sh import cloudfoundry_service_key.bucket_creds 483a6ac5-4ba0-48ad-9850-ef87b51aaa08
./run.sh plan
else
echo "Not importing bootstrap state"
fi
terraform/bootstrap/main.tf
+24
View File
@@ -0,0 +1,24 @@
locals {
cf_api_url = "https://api.fr.cloud.gov"
s3_service_name = "notify-terraform-state"
}
module "s3" {
source = "github.com/18f/terraform-cloudgov//s3"
cf_api_url = local.cf_api_url
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = "gsa-10x-prototyping"
cf_space_name = "10x-notifications"
s3_service_name = local.s3_service_name
}
resource "cloudfoundry_service_key" "bucket_creds" {
name = "${local.s3_service_name}-access"
service_instance = module.s3.bucket_id
}
output "bucket_credentials" {
value = cloudfoundry_service_key.bucket_creds.credentials
}
terraform/bootstrap/providers.tf
+16
View File
@@ -0,0 +1,16 @@
terraform {
required_version = "~> 1.0"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.15.5"
}
}
}
provider "cloudfoundry" {
api_url = local.cf_api_url
user = var.cf_user
password = var.cf_password
app_logs_max = 30
}
terraform/bootstrap/run.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
if [[ ! -f "secrets.auto.tfvars" ]]; then
../create_service_account.sh -s 10x-notifications -u config-bootstrap-deployer > secrets.auto.tfvars
fi
if [[ $# -gt 0 ]]; then
echo "Running terraform $@"
terraform $@
else
echo "Not running terraform"
fi
terraform/bootstrap/teardown_creds.sh
Executable
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
../destroy_service_account.sh -s 10x-notifications -u config-bootstrap-deployer
rm secrets.auto.tfvars
terraform/bootstrap/variables.tf
+2
View File
@@ -0,0 +1,2 @@
variable "cf_password" {}
variable "cf_user" {}
terraform/create_service_account.sh
Executable
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
org="gsa-10x-prototyping"
usage="
$0: Create a Service User Account for a given space
Usage:
$0 -h
$0 -s <SPACE NAME> -u <USER NAME> [-r <ROLE NAME>] [-o <ORG NAME>]
Options:
-h: show help and exit
-s <SPACE NAME>: configure the space to act on. Required
-u <USER NAME>: set the service user name. Required
-r <ROLE NAME>: set the service user's role to either space-deployer or space-auditor. Default: space-deployer
-o <ORG NAME>: configure the organization to act on. Default: $org
"
set -e
set -o pipefail
space=""
service=""
role="space-deployer"
while getopts ":hs:u:r:o:" opt; do
case "$opt" in
s)
space=${OPTARG}
;;
u)
service=${OPTARG}
;;
r)
role=${OPTARG}
;;
o)
org=${OPTARG}
;;
h)
echo "$usage"
exit 0
;;
esac
done
if [[ $space = "" || $service = "" ]]; then
echo "$usage"
exit 1
fi
cf target -o $org -s $space 1>&2
# create user account service
cf create-service cloud-gov-service-account $role $service 1>&2
# create service key
cf create-service-key $service service-account-key 1>&2
# output service key to stdout in secrets.auto.tfvars format
creds=`cf service-key $service service-account-key | tail -n 4`
username=`echo $creds | jq '.username'`
password=`echo $creds | jq '.password'`
cat << EOF
# generated with $0 -s $space -u $service -r $role -o $org
# revoke with $(dirname $0)/destroy_service_account.sh -s $space -u $service -o $org
cf_user = $username
cf_password = $password
EOF
terraform/destroy_service_account.sh
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
org="gsa-10x-prototyping"
usage="
$0: Destroy a Service User Account in a given space
Usage:
$0 -h
$0 -s <SPACE NAME> -u <USER NAME> [-o <ORG NAME>]
Options:
-h: show help and exit
-s <SPACE NAME>: configure the space to act on. Required
-u <USER NAME>: configure the service user name to destroy. Required
-o <ORG NAME>: configure the organization to act on. Default: $org
"
set -e
space=""
service=""
while getopts ":hs:u:o:" opt; do
case "$opt" in
s)
space=${OPTARG}
;;
u)
service=${OPTARG}
;;
o)
org=${OPTARG}
;;
h)
echo "$usage"
exit 0
;;
esac
done
if [[ $space = "" || $service = "" ]]; then
echo "$usage"
exit 1
fi
cf target -o $org -s $space
# destroy service key
cf delete-service-key $service service-account-key -f
# destroy service
cf delete-service $service -f
terraform/production/main.tf
+54
View File
@@ -0,0 +1,54 @@
locals {
cf_org_name = "TKTK"
cf_space_name = "TKTK"
env = "production"
app_name = "notifications-api"
recursive_delete = false
}
module "database" {
source = "github.com/18f/terraform-cloudgov//database"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
env = local.env
app_name = local.app_name
recursive_delete = local.recursive_delete
rds_plan_name = "TKTK-production-rds-plan"
}
module "redis" {
source = "github.com/18f/terraform-cloudgov//redis"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
env = local.env
app_name = local.app_name
recursive_delete = local.recursive_delete
redis_plan_name = "TKTK-production-redis-plan"
}
###########################################################################
# The following lines need to be commented out for the initial `terraform apply`
# It can be re-enabled after:
# 1) the app has first been deployed
# 2) the route has been manually created by an OrgManager:
# `cf create-domain TKTK-org-name TKTK-production-domain-name`
###########################################################################
# module "domain" {
# source = "github.com/18f/terraform-cloudgov//domain"
#
# cf_user = var.cf_user
# cf_password = var.cf_password
# cf_org_name = local.cf_org_name
# cf_space_name = local.cf_space_name
# env = local.env
# app_name = local.app_name
# recursive_delete = local.recursive_delete
# cdn_plan_name = "domain"
# domain_name = "TKTK-production-domain-name"
# }
terraform/production/providers.tf
+17
View File
@@ -0,0 +1,17 @@
terraform {
required_version = "~> 1.0"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.15.5"
}
}
backend "s3" {
bucket = "cg-31204bcc-aae3-4cd3-8b59-5055a338d44f"
key = "api.tfstate.prod"
encrypt = "true"
region = "us-gov-west-1"
profile = "notify-terraform-backend"
}
}
terraform/production/variables.tf
+2
View File
@@ -0,0 +1,2 @@
variable "cf_password" {}
variable "cf_user" {}
terraform/set_space_egress.sh
Executable
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
org="gsa-10x-prototyping"
usage="
$0: Set egress rules for given space
Usage:
$0 -h
$0 -s <SPACE NAME> [-o <ORG NAME>] [-p] [-t]
Options:
-h: show help and exit
-s <SPACE NAME>: configure the space to act on. Required
-o <ORG NAME>: configure the organization to act on. Default: $org
-p: Add the public egress rules
-t: Add the trusted egress rules
Notes:
* If -p or -t are not passed, the related security groups will be removed, if they were present
"
set -e
space=""
public=false
trusted=false
while getopts ":hs:o:pt" opt; do
case "$opt" in
s)
space=${OPTARG}
;;
o)
org=${OPTARG}
;;
p)
public=true
;;
t)
trusted=true
;;
h)
echo "$usage"
exit 0
;;
esac
done
if [[ $space = "" ]]; then
echo "$usage"
exit 1
fi
if [[ $public = true ]]; then
cf bind-security-group public_networks_egress $org --space $space
else
cf unbind-security-group public_networks_egress $org $space
fi
if [[ $trusted = true ]]; then
cf bind-security-group trusted_local_networks_egress $org --space $space
else
cf unbind-security-group trusted_local_networks_egress $org $space
fi
echo "Done"
terraform/staging/main.tf
+55
View File
@@ -0,0 +1,55 @@
locals {
cf_org_name = "gsa-10x-prototyping"
cf_space_name = "10x-notifications"
env = "staging"
app_name = "notifications-api"
recursive_delete = true
}
module "database" {
source = "github.com/18f/terraform-cloudgov//database"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
env = local.env
app_name = local.app_name
recursive_delete = local.recursive_delete
rds_plan_name = "micro-psql"
}
module "redis" {
source = "github.com/18f/terraform-cloudgov//redis"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
env = local.env
app_name = local.app_name
recursive_delete = local.recursive_delete
redis_plan_name = "redis-dev"
}
module "csv_upload_bucket" {
source = "github.com/18f/terraform-cloudgov//s3"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
recursive_delete = local.recursive_delete
s3_service_name = "${local.app_name}-csv-upload-bucket-${local.env}"
}
module "contact_list_bucket" {
source = "github.com/18f/terraform-cloudgov//s3"
cf_user = var.cf_user
cf_password = var.cf_password
cf_org_name = local.cf_org_name
cf_space_name = local.cf_space_name
recursive_delete = local.recursive_delete
s3_service_name = "${local.app_name}-contact-list-bucket-${local.env}"
}
terraform/staging/providers.tf
+17
View File
@@ -0,0 +1,17 @@
terraform {
required_version = "~> 1.0"
required_providers {
cloudfoundry = {
source = "cloudfoundry-community/cloudfoundry"
version = "0.15.5"
}
}
backend "s3" {
bucket = "cg-31204bcc-aae3-4cd3-8b59-5055a338d44f"
key = "api.tfstate.stage"
encrypt = "true"
region = "us-gov-west-1"
profile = "notify-terraform-backend"
}
}
terraform/staging/variables.tf
+2
View File
@@ -0,0 +1,2 @@
variable "cf_password" {}
variable "cf_user" {}
tests/app/aws/test_s3.py
+11 -2
View File
@@ -4,7 +4,13 @@ import pytest
import pytz
from freezegun import freeze_time
from app.aws.s3 import get_list_of_files_by_suffix, get_s3_file
from app.aws.s3 import (
default_access_key,
default_region,
default_secret_key,
get_list_of_files_by_suffix,
get_s3_file,
)
from tests.app.conftest import datetime_in_past
@@ -22,7 +28,10 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
get_s3_mock.assert_called_with(
'foo-bucket',
'bar-file.txt'
'bar-file.txt',
default_access_key,
default_secret_key,
default_region,
)
tests/app/celery/test_ftp_update_tasks.py
+5 -1
View File
@@ -1,3 +1,4 @@
import os
from collections import defaultdict, namedtuple
from datetime import date, datetime
@@ -90,7 +91,10 @@ def test_update_letter_notifications_statuses_calls_with_correct_bucket_location
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
s3_mock.assert_called_with('{}-ftp'.format(
current_app.config['NOTIFY_EMAIL_DOMAIN']),
'NOTIFY-20170823160812-RSP.TXT'
'NOTIFY-20170823160812-RSP.TXT',
os.environ['AWS_ACCESS_KEY_ID'],
os.environ['AWS_SECRET_ACCESS_KEY'],
os.environ['AWS_REGION'],
)
tests/app/celery/test_nightly_tasks.py
+3
View File
@@ -319,6 +319,7 @@ def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_friday_
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match)
letter_raise_alert_if_no_ack_file_for_zip()
@@ -334,6 +335,7 @@ def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_mat
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
@@ -360,6 +362,7 @@ def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db_se
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
letter_raise_alert_if_no_ack_file_for_zip()
tests/app/celery/test_research_mode_tasks.py
+5
View File
@@ -134,6 +134,7 @@ def test_failure_firetext_callback(phone_number):
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_uploads_response_file_s3(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
@@ -157,6 +158,7 @@ def test_create_fake_letter_response_file_uploads_response_file_s3(
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_calls_dvla_callback_on_development(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
@@ -193,6 +195,7 @@ def test_create_fake_letter_response_file_calls_dvla_callback_on_development(
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_does_not_call_dvla_callback_on_preview(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
@@ -208,6 +211,7 @@ def test_create_fake_letter_response_file_does_not_call_dvla_callback_on_preview
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_tries_to_create_files_with_other_filenames(notify_api, mocker):
mock_file_exists = mocker.patch('app.celery.research_mode_tasks.file_exists', side_effect=[True, True, False])
mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
@@ -228,6 +232,7 @@ def test_create_fake_letter_response_file_tries_to_create_files_with_other_filen
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_gives_up_after_thirty_times(notify_api, mocker):
mock_file_exists = mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=True)
mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
tests/app/celery/test_scheduled_tasks.py
+4
View File
@@ -366,6 +366,7 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_restarts_scan_for_stuck_letters(
mocker,
sample_letter_template
@@ -396,6 +397,7 @@ def test_check_if_letters_still_pending_virus_check_restarts_scan_for_stuck_lett
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_raises_zendesk_if_files_cant_be_found(
mocker,
sample_letter_template
@@ -447,6 +449,7 @@ def test_check_if_letters_still_pending_virus_check_raises_zendesk_if_files_cant
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
@@ -481,6 +484,7 @@ def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_temp
@freeze_time("2019-01-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
+5
View File
@@ -28,6 +28,7 @@ from tests.app.db import (
@mock_s3
@freeze_time('2019-09-01 04:30')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_move_notifications_deletes_letters_from_s3(sample_letter_template, mocker):
s3 = boto3.client('s3', region_name='eu-west-1')
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
@@ -53,6 +54,7 @@ def test_move_notifications_deletes_letters_from_s3(sample_letter_template, mock
@mock_s3
@freeze_time('2019-09-01 04:30')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_move_notifications_copes_if_letter_not_in_s3(sample_letter_template, mocker):
s3 = boto3.client('s3', region_name='eu-west-1')
s3.create_bucket(
@@ -91,6 +93,7 @@ def test_move_notifications_does_nothing_if_notification_history_row_already_exi
@pytest.mark.parametrize(
'notification_status', ['validation-failed', 'virus-scan-failed']
)
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_move_notifications_deletes_letters_not_sent_and_in_final_state_from_table_but_not_s3(
sample_service, mocker, notification_status
):
@@ -115,6 +118,7 @@ def test_move_notifications_deletes_letters_not_sent_and_in_final_state_from_tab
@mock_s3
@freeze_time('2020-12-24 04:30')
@pytest.mark.parametrize('notification_status', ['delivered', 'returned-letter', 'technical-failure'])
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_move_notifications_deletes_letters_sent_and_in_final_state_from_table_and_s3(
sample_service, mocker, notification_status
):
@@ -153,6 +157,7 @@ def test_move_notifications_deletes_letters_sent_and_in_final_state_from_table_a
@pytest.mark.parametrize('notification_status', ['pending-virus-check', 'created', 'sending'])
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_move_notifications_does_not_delete_letters_not_yet_in_final_state(
sample_service, mocker, notification_status
):
tests/app/letters/test_letter_utils.py
+1
View File
@@ -31,6 +31,7 @@ from tests.app.db import create_notification
FROZEN_DATE_TIME = "2018-03-14 17:00:00"
pytest.skip(reason="Skipping letter-related functionality for now", allow_module_level=True)
@pytest.fixture(name='sample_precompiled_letter_notification')
def _sample_precompiled_letter_notification(sample_letter_notification):
tests/app/service/send_notification/test_send_notification.py
+3 -3
View File
@@ -683,7 +683,7 @@ def test_should_persist_notification(
(SMS_TYPE, 'send-sms-tasks'),
(EMAIL_TYPE, 'send-email-tasks')
])
def test_should_delete_notification_and_return_error_if_sqs_fails(
def test_should_delete_notification_and_return_error_if_redis_fails(
client,
sample_email_template,
sample_template,
@@ -694,7 +694,7 @@ def test_should_delete_notification_and_return_error_if_sqs_fails(
):
mocked = mocker.patch(
'app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type),
side_effect=Exception("failed to talk to SQS")
side_effect=Exception("failed to talk to redis")
)
mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid)
@@ -719,7 +719,7 @@ def test_should_delete_notification_and_return_error_if_sqs_fails(
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]
)
assert str(e.value) == 'failed to talk to SQS'
assert str(e.value) == 'failed to talk to redis'
mocked.assert_called_once_with([fake_uuid], queue=queue_name)
assert not notifications_dao.get_notification_by_id(fake_uuid)
tests/app/service/send_notification/test_send_pdf_letter_notification.py
+2
View File
@@ -71,6 +71,7 @@ def test_send_pdf_letter_notification_raises_error_if_service_in_trial_mode(
assert 'trial mode' in e.value.message
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_send_pdf_letter_notification_raises_error_when_pdf_is_not_in_transient_letter_bucket(
mocker,
sample_service_full_permissions,
@@ -97,6 +98,7 @@ def test_send_pdf_letter_notification_does_nothing_if_notification_already_exist
@freeze_time("2019-08-02 11:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_send_pdf_letter_notification_creates_notification_and_moves_letter(
mocker,
sample_service_full_permissions,
tests/app/service/test_rest.py
+1
View File
@@ -2514,6 +2514,7 @@ def test_send_one_off_notification(sample_service, admin_request, mocker):
assert response['id'] == str(noti.id)
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_pdf_letter(mocker, sample_service_full_permissions, client, fake_uuid, notify_user):
mocker.patch('app.service.send_notification.utils_s3download')
mocker.patch('app.service.send_notification.get_page_count', return_value=1)
tests/app/test_cloudfoundry_config.py
+25 -2
View File
@@ -16,16 +16,39 @@ def vcap_services():
}],
'aws-elasticache-redis': [{
'credentials': {
'uri': 'redis uri'
'uri': 'redis://xxx:6379'
}
}],
's3': [
{
'name': 'notifications-api-csv-upload-bucket-test',
'credentials': {
'access_key_id': 'csv-access',
'bucket': 'csv-upload-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'csv-secret'
}
},
{
'name': 'notifications-api-contact-list-bucket-test',
'credentials': {
'access_key_id': 'contact-access',
'bucket': 'contact-list-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'contact-secret'
}
}
],
'user-provided': []
}
def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
os.environ['DEPLOY_ENV'] = 'test'
os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
extract_cloudfoundry_config()
assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri'
assert os.environ['REDIS_URL'] == 'redis uri'
assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket'
assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket'