mirror of
https://github.com/GSA/notifications-api.git
synced 2026-08-21 06:49:26 -04:00
Compare commits
1 Commits
7-22-2025
...
change-err
| Author | SHA1 | Date | |
|---|---|---|---|
|
0ad2ebc4b3 |
109
.cfignore
109
.cfignore
@@ -1,109 +0,0 @@
|
||||
# from deploy-exclude.lst
|
||||
|
||||
*__pycache__*
|
||||
.git/*
|
||||
app/assets/*
|
||||
bower_components/*
|
||||
cache/*
|
||||
.cache/*
|
||||
node_modules/*
|
||||
target/*
|
||||
venv/*
|
||||
build/*
|
||||
.envrc
|
||||
tests/.cache/*
|
||||
.cf/*
|
||||
|
||||
# from .gitignore
|
||||
|
||||
queues.csv
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
.venv/
|
||||
venv/
|
||||
venv-freeze/
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
/cache
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
.pytest_cache
|
||||
coverage.xml
|
||||
test_results.xml
|
||||
*,cover
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
.idea/
|
||||
.vscode
|
||||
|
||||
# Mac
|
||||
*.DS_Store
|
||||
environment.sh
|
||||
.envrc
|
||||
.env
|
||||
.env*
|
||||
varsfile
|
||||
|
||||
celerybeat-schedule
|
||||
|
||||
# CloudFoundry
|
||||
.cf
|
||||
varsfile*
|
||||
.secret*
|
||||
|
||||
/scripts/run_my_tests.sh
|
||||
|
||||
# Terraform
|
||||
.terraform.lock.hcl
|
||||
**/.terraform/*
|
||||
secrets.auto.tfvars
|
||||
terraform.tfstate
|
||||
terraform.tfstate.backup
|
||||
@@ -1,5 +0,0 @@
|
||||
[report]
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
^\s*logger\.
|
||||
^\s*current_app\.logger\.
|
||||
378
.ds.baseline
378
.ds.baseline
@@ -1,378 +0,0 @@
|
||||
{
|
||||
"version": "1.5.0",
|
||||
"plugins_used": [
|
||||
{
|
||||
"name": "ArtifactoryDetector"
|
||||
},
|
||||
{
|
||||
"name": "AWSKeyDetector"
|
||||
},
|
||||
{
|
||||
"name": "AzureStorageKeyDetector"
|
||||
},
|
||||
{
|
||||
"name": "Base64HighEntropyString",
|
||||
"limit": 4.5
|
||||
},
|
||||
{
|
||||
"name": "BasicAuthDetector"
|
||||
},
|
||||
{
|
||||
"name": "CloudantDetector"
|
||||
},
|
||||
{
|
||||
"name": "DiscordBotTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "GitHubTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "GitLabTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "HexHighEntropyString",
|
||||
"limit": 3.0
|
||||
},
|
||||
{
|
||||
"name": "IbmCloudIamDetector"
|
||||
},
|
||||
{
|
||||
"name": "IbmCosHmacDetector"
|
||||
},
|
||||
{
|
||||
"name": "IPPublicDetector"
|
||||
},
|
||||
{
|
||||
"name": "JwtTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "KeywordDetector",
|
||||
"keyword_exclude": ""
|
||||
},
|
||||
{
|
||||
"name": "MailchimpDetector"
|
||||
},
|
||||
{
|
||||
"name": "NpmDetector"
|
||||
},
|
||||
{
|
||||
"name": "OpenAIDetector"
|
||||
},
|
||||
{
|
||||
"name": "PrivateKeyDetector"
|
||||
},
|
||||
{
|
||||
"name": "PypiTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "SendGridDetector"
|
||||
},
|
||||
{
|
||||
"name": "SlackDetector"
|
||||
},
|
||||
{
|
||||
"name": "SoftlayerDetector"
|
||||
},
|
||||
{
|
||||
"name": "SquareOAuthDetector"
|
||||
},
|
||||
{
|
||||
"name": "StripeDetector"
|
||||
},
|
||||
{
|
||||
"name": "TelegramBotTokenDetector"
|
||||
},
|
||||
{
|
||||
"name": "TwilioKeyDetector"
|
||||
}
|
||||
],
|
||||
"filters_used": [
|
||||
{
|
||||
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.common.is_baseline_file",
|
||||
"filename": ".ds.baseline"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
|
||||
"min_level": 2
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_indirect_reference"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_likely_id_string"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_lock_file"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_potential_uuid"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_sequential_string"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_swagger_file"
|
||||
},
|
||||
{
|
||||
"path": "detect_secrets.filters.heuristic.is_templated_secret"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
".github/workflows/checks.yml": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": ".github/workflows/checks.yml",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 28,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Basic Auth Credentials",
|
||||
"filename": ".github/workflows/checks.yml",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 45,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
".github/workflows/daily_checks.yml": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": ".github/workflows/daily_checks.yml",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 63,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Basic Auth Credentials",
|
||||
"filename": ".github/workflows/daily_checks.yml",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 79,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"app/enums.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "app/enums.py",
|
||||
"hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3",
|
||||
"is_verified": false,
|
||||
"line_number": 123,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"app/notifications/receive_notifications.py": [
|
||||
{
|
||||
"type": "Base64 High Entropy String",
|
||||
"filename": "app/notifications/receive_notifications.py",
|
||||
"hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
|
||||
"is_verified": false,
|
||||
"line_number": 29,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"deploy-config/sandbox.yml": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "deploy-config/sandbox.yml",
|
||||
"hashed_secret": "113151dd10316fcb0d5507b6215d78e2f3fe9e54",
|
||||
"is_verified": false,
|
||||
"line_number": 11,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"sample.env": [
|
||||
{
|
||||
"type": "Basic Auth Credentials",
|
||||
"filename": "sample.env",
|
||||
"hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
|
||||
"is_verified": false,
|
||||
"line_number": 16,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/clients/test_document_download.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/clients/test_document_download.py",
|
||||
"hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
|
||||
"is_verified": false,
|
||||
"line_number": 14,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/clients/test_performance_platform.py": [
|
||||
{
|
||||
"type": "Base64 High Entropy String",
|
||||
"filename": "tests/app/clients/test_performance_platform.py",
|
||||
"hashed_secret": "76bb66c38ac4046bf73cd4a2c35a2b0af94aeb61",
|
||||
"is_verified": false,
|
||||
"line_number": 84,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/dao/test_services_dao.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/dao/test_services_dao.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 290,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/dao/test_users_dao.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/dao/test_users_dao.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 69,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/dao/test_users_dao.py",
|
||||
"hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
|
||||
"is_verified": false,
|
||||
"line_number": 199,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/db.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/db.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 90,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/notifications/test_receive_notification.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/notifications/test_receive_notification.py",
|
||||
"hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
|
||||
"is_verified": false,
|
||||
"line_number": 28,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Base64 High Entropy String",
|
||||
"filename": "tests/app/notifications/test_receive_notification.py",
|
||||
"hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
|
||||
"is_verified": false,
|
||||
"line_number": 58,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/notifications/test_validators.py": [
|
||||
{
|
||||
"type": "Base64 High Entropy String",
|
||||
"filename": "tests/app/notifications/test_validators.py",
|
||||
"hashed_secret": "6c1a8443963d02d13ffe575a71abe19ea731fb66",
|
||||
"is_verified": false,
|
||||
"line_number": 672,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/service/test_rest.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/service/test_rest.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 1288,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/test_cloudfoundry_config.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/test_cloudfoundry_config.py",
|
||||
"hashed_secret": "e5e178db7317356946d13e5d2da037d39ac61c71",
|
||||
"is_verified": false,
|
||||
"line_number": 12,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Basic Auth Credentials",
|
||||
"filename": "tests/app/test_cloudfoundry_config.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 14,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/test_cloudfoundry_config.py",
|
||||
"hashed_secret": "cfd48edeb81ba7d48cbddcf1eeede25ba67057e8",
|
||||
"is_verified": false,
|
||||
"line_number": 33,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/app/user/test_rest.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/user/test_rest.py",
|
||||
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
|
||||
"is_verified": false,
|
||||
"line_number": 110,
|
||||
"is_secret": false
|
||||
},
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/app/user/test_rest.py",
|
||||
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
|
||||
"is_verified": false,
|
||||
"line_number": 874,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/notifications_utils/clients/antivirus/test_antivirus_client.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/notifications_utils/clients/antivirus/test_antivirus_client.py",
|
||||
"hashed_secret": "932b25270abe1301c22c709a19082dff07d469ff",
|
||||
"is_verified": false,
|
||||
"line_number": 16,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/notifications_utils/clients/encryption/test_encryption_client.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/notifications_utils/clients/encryption/test_encryption_client.py",
|
||||
"hashed_secret": "f1e923a9667de11be6a210849a8651c1bfd81605",
|
||||
"is_verified": false,
|
||||
"line_number": 13,
|
||||
"is_secret": false
|
||||
}
|
||||
],
|
||||
"tests/notifications_utils/clients/zendesk/test_zendesk_client.py": [
|
||||
{
|
||||
"type": "Secret Keyword",
|
||||
"filename": "tests/notifications_utils/clients/zendesk/test_zendesk_client.py",
|
||||
"hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
|
||||
"is_verified": false,
|
||||
"line_number": 16,
|
||||
"is_secret": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"generated_at": "2025-07-02T18:56:01Z"
|
||||
}
|
||||
7
.flake8
Normal file
7
.flake8
Normal file
@@ -0,0 +1,7 @@
|
||||
[flake8]
|
||||
# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
|
||||
# W503: line break before binary operator
|
||||
exclude = venv*,__pycache__,node_modules,cache,migrations,build
|
||||
ignore = W503
|
||||
max-complexity = 14
|
||||
max-line-length = 120
|
||||
1
.github/ISSUE_TEMPLATE/config.yml
vendored
1
.github/ISSUE_TEMPLATE/config.yml
vendored
@@ -1 +0,0 @@
|
||||
blank_issues_enabled: true
|
||||
63
.github/ISSUE_TEMPLATE/create-new-adr-form.yml
vendored
63
.github/ISSUE_TEMPLATE/create-new-adr-form.yml
vendored
@@ -1,63 +0,0 @@
|
||||
name: Create a new ADR
|
||||
description: Create a new issue with an ADR proposal
|
||||
labels: ["ADR: proposed", "documentation"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Use this form to create a new Architectural Decision Record.
|
||||
- type: textarea
|
||||
id: context
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Context
|
||||
description: >
|
||||
Explain the nature of the problem, challenge, or decision to be made here.
|
||||
Be as succinct as possible, but do not leave out any details. State things
|
||||
matter of factly and as objectively as you can.
|
||||
- type: textarea
|
||||
id: decision
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Decision
|
||||
description: >
|
||||
What was decided? This should be an active-voice statement. For example,
|
||||
"We will use the US Web Design System." Note any major options that were
|
||||
not selected.
|
||||
- type: textarea
|
||||
id: consequences
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: Consequences
|
||||
description: >
|
||||
What are the consequences of this decision? Consequences can be
|
||||
positive, negative, or neutral, but all known consequences of this
|
||||
decision should be listed here.
|
||||
- type: input
|
||||
id: author
|
||||
attributes:
|
||||
label: Author
|
||||
description: Who is the author(s) of this ADR?
|
||||
placeholder: Enter your GitHub username here, and anyone else helping author this
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: stakeholders
|
||||
attributes:
|
||||
label: Stakeholders
|
||||
description: Who are the stakeholders of this ADR?
|
||||
placeholder: Enter their GitHub username(s) here
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: next-steps
|
||||
attributes:
|
||||
label: Next Steps
|
||||
description: >
|
||||
Once option has already been chosen and justified, summarize or list the next steps
|
||||
that came from the decision. Link to GitHub issues that track these steps.
|
||||
validations:
|
||||
required: false
|
||||
58
.github/actions/deploy-proxy/action.yml
vendored
58
.github/actions/deploy-proxy/action.yml
vendored
@@ -1,58 +0,0 @@
|
||||
name: Deploy egress proxy
|
||||
description: Set egress space security groups and deploy proxy
|
||||
inputs:
|
||||
cf_org:
|
||||
description: The org the target app exists in.
|
||||
required: true
|
||||
cf_space:
|
||||
description: The space the target app exists in.
|
||||
required: true
|
||||
app:
|
||||
description: application name to be proxied.
|
||||
required: true
|
||||
proxy_repo:
|
||||
description: git repo for cg-egress-proxy
|
||||
default: https://github.com/GSA-TTS/cg-egress-proxy.git
|
||||
proxy_version:
|
||||
description: git ref to be deployed
|
||||
default: main
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install cf-cli
|
||||
shell: bash
|
||||
run: |
|
||||
curl -A "cg-deploy-action" -v -L -o cf-cli_amd64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=v8&source=github'
|
||||
sudo dpkg -i cf-cli_amd64.deb
|
||||
- name: Login to cf-cli
|
||||
shell: bash
|
||||
run: |
|
||||
cf api api.fr.cloud.gov
|
||||
cf auth
|
||||
- name: Target org and space
|
||||
shell: bash
|
||||
run: cf target -o ${{ inputs.cf_org }} -s ${{ inputs.cf_space }}
|
||||
- name: Set restricted space egress
|
||||
shell: bash
|
||||
run: ./terraform/set_space_egress.sh -t -s ${{ inputs.cf_space }}
|
||||
- name: Set public space egress
|
||||
shell: bash
|
||||
run: ./terraform/set_space_egress.sh -p -s ${{ inputs.cf_space }}-egress
|
||||
- name: Create temp directory
|
||||
shell: bash
|
||||
id: create-temp-dir
|
||||
run: echo "path=$(mktemp -d -t egress-XXXXXXXXXX --tmpdir=$RUNNER_TEMP)" >> $GITHUB_OUTPUT
|
||||
- name: Clone cg-egress-proxy
|
||||
shell: bash
|
||||
run: git clone ${{ inputs.proxy_repo }} ${{ steps.create-temp-dir.outputs.path }}
|
||||
- name: Switch to deploy ref
|
||||
shell: bash
|
||||
working-directory: ${{ steps.create-temp-dir.outputs.path }}
|
||||
run: git checkout ${{ inputs.proxy_version }}
|
||||
- name: Copy config files
|
||||
shell: bash
|
||||
run: cp ./deploy-config/egress_proxy/${{ inputs.app }}.*.acl ${{ steps.create-temp-dir.outputs.path }}
|
||||
- name: Deploy proxy
|
||||
shell: bash
|
||||
working-directory: ${{ steps.create-temp-dir.outputs.path }}
|
||||
run: ./bin/cf-deployproxy -a ${{ inputs.app }} -p egress-proxy -e egress_proxy
|
||||
24
.github/actions/setup-project/action.yml
vendored
24
.github/actions/setup-project/action.yml
vendored
@@ -1,24 +0,0 @@
|
||||
name: Set up project
|
||||
description: Setup python & install dependencies
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install container dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update \
|
||||
&& sudo apt-get install -y --no-install-recommends \
|
||||
libcurl4-openssl-dev
|
||||
- name: Set up Python 3.12.9
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.12.9"
|
||||
- name: Install poetry
|
||||
shell: bash
|
||||
run: pip install poetry==2.1.3
|
||||
- name: Install poetry export
|
||||
shell: bash
|
||||
run: poetry self add poetry-plugin-export
|
||||
- name: Downgrade virtualenv to compatible version
|
||||
shell: bash
|
||||
run: pip install "virtualenv<20.30"
|
||||
15
.github/dependabot.yml
vendored
15
.github/dependabot.yml
vendored
@@ -1,15 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "daily"
|
||||
labels:
|
||||
- "dependabot" # Custom label to identify Dependabot PRs
|
||||
assignees:
|
||||
- "alexjanousekGSA"
|
||||
22
.github/pull_request_template.md
vendored
22
.github/pull_request_template.md
vendored
@@ -1,22 +0,0 @@
|
||||
<!--
|
||||
Not sure what you should include or write in a pull request? Please read the
|
||||
[pull request documentation in our docs!](https://github.com/GSA/notifications-api/blob/main/docs/all.md#pull-requests)
|
||||
-->
|
||||
|
||||
*A note to PR reviewers: it may be helpful to review our [code review documentation](https://github.com/GSA/notifications-api/blob/main/docs/all.md#code-reviews) to know what to keep in mind while reviewing pull requests.*
|
||||
|
||||
## Description
|
||||
|
||||
Please enter a detailed description here.
|
||||
|
||||
## TODO (optional)
|
||||
|
||||
* [ ] TODO item 1
|
||||
* [ ] TODO item 2
|
||||
* [ ] TODO item ...
|
||||
|
||||
## Security Considerations
|
||||
|
||||
* Consideration 1
|
||||
* Consideration 2
|
||||
* Consideration ...
|
||||
83
.github/workflows/adr-accepted.yml
vendored
83
.github/workflows/adr-accepted.yml
vendored
@@ -1,83 +0,0 @@
|
||||
name: ADR accepted
|
||||
|
||||
on:
|
||||
issues:
|
||||
types:
|
||||
- closed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
accept:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: check for tags
|
||||
if: "${{ !contains(github.event.issue.labels.*.name, 'ADR: accepted' )}}"
|
||||
shell: bash
|
||||
run: exit 0
|
||||
|
||||
- name: checkout main branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
- name: get ADR number
|
||||
id: next
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p docs/adrs
|
||||
LAST_ADR=$(ls docs/adrs/*.md | grep -Eo "/[0-9]+-" | sort | tail -n1 | grep -Eo "[0-9]+")
|
||||
LAST_ADR=$(echo "$LAST_ADR" | sed -E 's/^0+//')
|
||||
NEXT_ADR=$(($LAST_ADR + 1))
|
||||
NEXT_ADR=$(printf "%04i" "$NEXT_ADR")
|
||||
echo "number=$NEXT_ADR" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: get date
|
||||
id: date
|
||||
shell: bash
|
||||
run: echo "date=$(date +'%B %d, %Y')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: build filename
|
||||
id: filename
|
||||
shell: bash
|
||||
run: |
|
||||
SLUG=$(printf '%q\n' "${{ github.event.issue.title }}" | tr A-Z a-z)
|
||||
SLUG=$(printf '%q\n' "$SLUG" | iconv -c -t ascii//TRANSLIT)
|
||||
SLUG=$(printf '%q\n' "$SLUG" | sed -E 's/[^a-z0-9]+/-/g' | sed -E 's/-+/-/g' | sed -E 's/^-+|-+$//g')
|
||||
|
||||
FILENAME="docs/adrs/${{ steps.next.outputs.number }}-$SLUG.md"
|
||||
echo "slug=$SLUG" >> "$GITHUB_OUTPUT"
|
||||
echo "filename=$FILENAME" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: write the ADR
|
||||
uses: DamianReeves/write-file-action@v1.3
|
||||
with:
|
||||
path: ${{ steps.filename.outputs.filename }}
|
||||
write-mode: overwrite
|
||||
contents: |
|
||||
# ${{ github.event.issue.title }}
|
||||
|
||||
Status: Accepted
|
||||
Date: ${{ steps.date.outputs.date }}
|
||||
|
||||
${{ github.event.issue.body }}
|
||||
|
||||
- name: branch, commit, and open PR
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH="adr/auto/${{ steps.filename.outputs.slug }}"
|
||||
git config --global user.email "tts@gsa.gov"
|
||||
git config --global user.name "Notify ADR Automation"
|
||||
git checkout -b $BRANCH
|
||||
git add docs/adrs/*.md
|
||||
git commit -m "add ADR ${{ steps.next.outputs.number }}: ${{ github.event.issue.title }}"
|
||||
git push -f origin $BRANCH
|
||||
gh pr create \
|
||||
--title "Add ADR ${{ steps.next.outputs.number }} to the repo" \
|
||||
--body "This pull request was opened automatically because #${{ github.event.issue.number }} was closed after being marked as an approved ADR. It contains a markdown file capturing the ADR body at the time the issue was closed. Please verify that the markdown is correct before merging!" || true
|
||||
gh pr merge $BRANCH --auto --squash || true
|
||||
147
.github/workflows/checks.yml
vendored
147
.github/workflows/checks.yml
vendored
@@ -1,147 +0,0 @@
|
||||
name: Run checks
|
||||
|
||||
on: [push]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
DEBUG: True
|
||||
NOTIFY_ENVIRONMENT: test
|
||||
NEW_RELIC_CONFIG_FILE: newrelic.ini
|
||||
NEW_RELIC_ENVIRONMENT: test
|
||||
FLASK_APP: application.py
|
||||
WERKZEUG_DEBUG_PIN: off
|
||||
REDIS_ENABLED: 0
|
||||
AWS_US_TOLL_FREE_NUMBER: "+18556438890"
|
||||
|
||||
jobs:
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_PASSWORD: password
|
||||
POSTGRES_DB: test_notification_api
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
# Maps tcp port 5432 on service container to the host
|
||||
- 5432:5432
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
|
||||
- name: Check imports alphabetized
|
||||
run: poetry run isort --check-only ./app ./tests
|
||||
- name: Run style checks
|
||||
run: poetry run flake8 .
|
||||
- name: Check for dead code
|
||||
run: make dead-code
|
||||
- name: Run tests with coverage
|
||||
run: poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
- name: Check coverage threshold
|
||||
# TODO get this back up to 95
|
||||
run: poetry run coverage report -m --fail-under=92
|
||||
|
||||
validate-new-relic-config:
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install poetry packages
|
||||
run: poetry install
|
||||
- name: Validate NewRelic config
|
||||
env:
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
# Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
|
||||
NEW_RELIC_ENVIRONMENT: staging
|
||||
run: poetry run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
|
||||
|
||||
pip-audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Create requirements.txt
|
||||
run: poetry export --output requirements.txt
|
||||
- uses: pypa/gh-action-pip-audit@v1.1.0
|
||||
with:
|
||||
inputs: requirements.txt
|
||||
ignore-vulns: |
|
||||
PYSEC-2023-312
|
||||
|
||||
static-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install bandit
|
||||
run: pip install bandit
|
||||
- name: Run scan
|
||||
run: bandit -r app/ --confidence-level medium
|
||||
|
||||
dynamic-scan:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_PASSWORD: password
|
||||
POSTGRES_DB: test_notification_api
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
# Maps tcp port 5432 on service container to the host
|
||||
- 5432:5432
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
- name: Run server
|
||||
run: make run-flask &
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
- name: Run OWASP API Scan
|
||||
uses: zaproxy/action-api-scan@v0.9.0
|
||||
with:
|
||||
docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
|
||||
target: 'http://localhost:6011/docs/openapi.yml'
|
||||
fail_action: true
|
||||
allow_issue_writing: false
|
||||
rules_file_name: 'zap.conf'
|
||||
cmd_options: '-I'
|
||||
93
.github/workflows/codeql.yml
vendored
93
.github/workflows/codeql.yml
vendored
@@ -1,93 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main", "production" ]
|
||||
pull_request:
|
||||
branches: [ "main", "production" ]
|
||||
schedule:
|
||||
- cron: '15 8 * * 2'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (GitHub.com only)
|
||||
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# required to fetch internal or private CodeQL packs
|
||||
packages: read
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: python
|
||||
build-mode: none
|
||||
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
||||
# Use `c-cpp` to analyze code written in C, C++ or both
|
||||
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
||||
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
||||
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
||||
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# If the analyze step fails for one of the languages you are analyzing with
|
||||
# "We were unable to automatically build your code", modify the matrix above
|
||||
# to set the build mode to "manual" for that language. Then modify this step
|
||||
# to build your code.
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
96
.github/workflows/daily_checks.yml
vendored
96
.github/workflows/daily_checks.yml
vendored
@@ -1,96 +0,0 @@
|
||||
name: Run daily scans
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# cron format: 'minute hour dayofmonth month dayofweek'
|
||||
# this will run at noon UTC every day (7am EST / 8am EDT)
|
||||
- cron: '0 12 * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
DEBUG: True
|
||||
NOTIFY_ENVIRONMENT: test
|
||||
NEW_RELIC_CONFIG_FILE: newrelic.ini
|
||||
NEW_RELIC_ENVIRONMENT: test
|
||||
FLASK_APP: application.py
|
||||
WERKZEUG_DEBUG_PIN: off
|
||||
REDIS_ENABLED: 0
|
||||
AWS_US_TOLL_FREE_NUMBER: "+18556438890"
|
||||
|
||||
jobs:
|
||||
pip-audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Create requirements.txt
|
||||
run: poetry export --output requirements.txt
|
||||
- uses: pypa/gh-action-pip-audit@v1.1.0
|
||||
with:
|
||||
inputs: requirements.txt
|
||||
ignore-vulns: |
|
||||
PYSEC-2023-312
|
||||
- name: Upload pip-audit artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: pip-audit-report
|
||||
path: /tmp/pip-audit-output.txt
|
||||
|
||||
static-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install bandit
|
||||
run: pip install bandit
|
||||
- name: Run scan
|
||||
run: bandit -r app/ -f txt -o /tmp/bandit-output.txt --confidence-level medium
|
||||
- name: Upload bandit artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bandit-report
|
||||
path: /tmp/bandit-output.txt
|
||||
|
||||
dynamic-scan:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_USER: user
|
||||
POSTGRES_PASSWORD: password
|
||||
POSTGRES_DB: test_notification_api
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
# Maps tcp port 5432 on service container to the host
|
||||
- 5432:5432
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
|
||||
NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
- name: Run server
|
||||
run: make run-flask &
|
||||
env:
|
||||
SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
|
||||
- name: Run OWASP API Scan
|
||||
uses: zaproxy/action-api-scan@v0.9.0
|
||||
with:
|
||||
docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
|
||||
target: 'http://localhost:6011/docs/openapi.yml'
|
||||
fail_action: true
|
||||
allow_issue_writing: false
|
||||
rules_file_name: 'zap.conf'
|
||||
cmd_options: '-I'
|
||||
105
.github/workflows/deploy-demo.yml
vendored
105
.github/workflows/deploy-demo.yml
vendored
@@ -1,105 +0,0 @@
|
||||
name: Deploy to demo environment
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ production ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
environment: demo
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform init
|
||||
working-directory: terraform/demo
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
- name: Terraform apply
|
||||
working-directory: terraform/demo
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform apply -auto-approve -input=false
|
||||
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
|
||||
- name: Create requirements.txt
|
||||
run: poetry export --output requirements.txt
|
||||
|
||||
- name: Deploy to cloud.gov
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-demo
|
||||
cf_command: >-
|
||||
push -f manifest.yml
|
||||
--vars-file deploy-config/demo.yml
|
||||
--var DANGEROUS_SALT="$DANGEROUS_SALT"
|
||||
--var SECRET_KEY="$SECRET_KEY"
|
||||
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
|
||||
--var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
|
||||
--var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
|
||||
--var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
|
||||
--var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
|
||||
--strategy rolling
|
||||
|
||||
- name: Update templates
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-demo
|
||||
cf_command: >-
|
||||
run-task notify-api-demo --command "flask command update-templates"
|
||||
|
||||
|
||||
- name: Deploy egress proxy
|
||||
uses: ./.github/actions/deploy-proxy
|
||||
env:
|
||||
CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
CF_PASSWORD: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
with:
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-demo
|
||||
app: notify-api-demo
|
||||
108
.github/workflows/deploy-prod.yml
vendored
108
.github/workflows/deploy-prod.yml
vendored
@@ -1,108 +0,0 @@
|
||||
name: Deploy to production environment
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ production ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# deploy-prod and deploy-demo will run in parallel now.
|
||||
# TODO: Research if we want to serialize them
|
||||
# by moving the jobs into a single file similar to
|
||||
# https://github.com/GSA/usnotify-ssb/blob/main/.github/workflows/apply.yml
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform init
|
||||
working-directory: terraform/production
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
- name: Terraform apply
|
||||
working-directory: terraform/production
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform apply -auto-approve -input=false
|
||||
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
|
||||
- name: Create requirements.txt
|
||||
run: poetry export --output requirements.txt
|
||||
|
||||
- name: Deploy to cloud.gov
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-production
|
||||
cf_command: >-
|
||||
push -f manifest.yml
|
||||
--vars-file deploy-config/production.yml
|
||||
--var DANGEROUS_SALT="$DANGEROUS_SALT"
|
||||
--var SECRET_KEY="$SECRET_KEY"
|
||||
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
|
||||
--var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
|
||||
--var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
|
||||
--var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
|
||||
--var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
|
||||
--strategy rolling
|
||||
|
||||
- name: Update templates
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-production
|
||||
cf_command: >-
|
||||
run-task notify-api-production --command "flask command update-templates"
|
||||
|
||||
- name: Deploy egress proxy
|
||||
uses: ./.github/actions/deploy-proxy
|
||||
env:
|
||||
CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
CF_PASSWORD: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
with:
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-production
|
||||
app: notify-api-production
|
||||
117
.github/workflows/deploy.yml
vendored
117
.github/workflows/deploy.yml
vendored
@@ -1,117 +0,0 @@
|
||||
name: Deploy to staging environment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [ Run checks ]
|
||||
types:
|
||||
- completed
|
||||
branches: [ main ] # Redundant, workflow_run events are only triggered on default branch (`main`)
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
|
||||
environment: staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform init
|
||||
working-directory: terraform/staging
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
- name: Terraform apply
|
||||
working-directory: terraform/staging
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform apply -auto-approve -input=false
|
||||
|
||||
- uses: ./.github/actions/setup-project
|
||||
- name: Install application dependencies
|
||||
run: make bootstrap
|
||||
|
||||
- name: Create requirements.txt
|
||||
run: poetry export --output requirements.txt
|
||||
|
||||
- name: Deploy to cloud.gov
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-staging
|
||||
cf_command: >-
|
||||
push -f manifest.yml
|
||||
--vars-file deploy-config/staging.yml
|
||||
--var DANGEROUS_SALT="$DANGEROUS_SALT"
|
||||
--var SECRET_KEY="$SECRET_KEY"
|
||||
--var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
|
||||
--var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
|
||||
--var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
|
||||
--var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
|
||||
--var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
|
||||
--strategy rolling
|
||||
|
||||
- name: Update templates
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
env:
|
||||
DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
|
||||
SECRET_KEY: ${{ secrets.SECRET_KEY }}
|
||||
ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
|
||||
NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
|
||||
NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
|
||||
NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
|
||||
LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE"
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-staging
|
||||
cf_command: >-
|
||||
run-task notify-api-staging --command "flask command update-templates"
|
||||
|
||||
- name: Deploy egress proxy
|
||||
uses: ./.github/actions/deploy-proxy
|
||||
env:
|
||||
CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
CF_PASSWORD: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
with:
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-staging
|
||||
app: notify-api-staging
|
||||
|
||||
bail:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event.workflow_run.conclusion == 'failure' }}
|
||||
steps:
|
||||
- uses: actions/github-script@v6
|
||||
with:
|
||||
script: core.setFailed('Checks failed, not deploying')
|
||||
123
.github/workflows/drift.yml
vendored
123
.github/workflows/drift.yml
vendored
@@ -1,123 +0,0 @@
|
||||
name: Verify Infrastructure
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# cron format: 'minute hour dayofmonth month dayofweek'
|
||||
# this will run at noon UTC every day (7am EST / 8am EDT)
|
||||
- cron: '0 12 * * *'
|
||||
|
||||
jobs:
|
||||
check_staging_drift:
|
||||
runs-on: ubuntu-latest
|
||||
name: Check for drift of staging terraform configuration
|
||||
environment: staging
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Check for drift
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: |
|
||||
cd terraform/staging
|
||||
terraform init
|
||||
terraform plan -detailed-exitcode
|
||||
exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "No changes detected. Intrastructure is up-to-date."
|
||||
elif [ $exit_code -eq 2 ]; then
|
||||
echo "Changes detected. Infrastructure drift found."
|
||||
exit 1
|
||||
else
|
||||
echo "Error running terraform plan."
|
||||
exit $exit_code
|
||||
fi
|
||||
|
||||
check_demo_drift:
|
||||
runs-on: ubuntu-latest
|
||||
name: Check for drift of demo terraform configuration
|
||||
environment: demo
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: 'production'
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Check for drift
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: |
|
||||
cd terraform/demo
|
||||
terraform init
|
||||
terraform plan -detailed-exitcode
|
||||
exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "No changes detected. Intrastructure is up-to-date."
|
||||
elif [ $exit_code -eq 2 ]; then
|
||||
echo "Changes detected. Infrastructure drift found."
|
||||
exit 1
|
||||
else
|
||||
echo "Error running terraform plan."
|
||||
exit $exit_code
|
||||
fi
|
||||
|
||||
check_prod_drift:
|
||||
runs-on: ubuntu-latest
|
||||
name: Check for drift of production terraform configuration
|
||||
environment: production
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: 'production'
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Check for drift
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: |
|
||||
cd terraform/production
|
||||
terraform init
|
||||
terraform plan -detailed-exitcode
|
||||
exit_code=$?
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "No changes detected. Intrastructure is up-to-date."
|
||||
elif [ $exit_code -eq 2 ]; then
|
||||
echo "Changes detected. Infrastructure drift found."
|
||||
exit 1
|
||||
else
|
||||
echo "Error running terraform plan."
|
||||
exit $exit_code
|
||||
fi
|
||||
68
.github/workflows/restage-apps.yml
vendored
68
.github/workflows/restage-apps.yml
vendored
@@ -1,68 +0,0 @@
|
||||
---
|
||||
name: Restage apps
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
environment:
|
||||
description: "Which environment needs to be restaged"
|
||||
required: true
|
||||
default: staging
|
||||
type: environment
|
||||
|
||||
jobs:
|
||||
restage_apps:
|
||||
runs-on: ubuntu-latest
|
||||
environment: ${{ inputs.environment }}
|
||||
strategy:
|
||||
matrix:
|
||||
app: ["api", "admin"]
|
||||
steps:
|
||||
- name: Update env variables api
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-${{ inputs.environment }}
|
||||
command: |
|
||||
cf set-env notify-api-${{inputs.environment}} DANGEROUS_SALT "${{secrets.DANGEROUS_SALT}}"
|
||||
cf set-env notify-api-${{inputs.environment}} ADMIN_CLIENT_SECRET "${{secrets.ADMIN_CLIENT_SECRET}}"
|
||||
cf set-env notify-api-${{inputs.environment}} SECRET_KEY "${{secrets.SECRET_KEY}}"
|
||||
cf set-env notify-api-${{inputs.environment}} LOGIN_PEM "${{secrets.LOGIN_PEM}}"
|
||||
cf set-env notify-api-${{inputs.environment}} NOTIFY_E2E_TEST_EMAIL "${{secrets.NOTIFY_E2E_TEST_EMAIL}}"
|
||||
cf set-env notify-api-${{inputs.environment}} NOTIFY_E2E_TEST_PASSWORD "${{secrets.NOTIFY_E2E_TEST_PASSWORD}}"
|
||||
|
||||
- name: Update env variables admin
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-${{ inputs.environment }}
|
||||
command: |
|
||||
cf set-env notify-admin-${{inputs.environment}} DANGEROUS_SALT "${{secrets.DANGEROUS_SALT}}"
|
||||
cf set-env notify-admin-${{inputs.environment}} ADMIN_CLIENT_SECRET "${{secrets.ADMIN_CLIENT_SECRET}}"
|
||||
cf set-env notify-admin-${{inputs.environment}} SECRET_KEY "${{secrets.SECRET_KEY}}"
|
||||
cf set-env notify-admin-${{inputs.environment}} LOGIN_PEM "${{secrets.LOGIN_PEM}}"
|
||||
cf set-env notify-admin-${{inputs.environment}} NOTIFY_E2E_TEST_EMAIL "${{secrets.NOTIFY_E2E_TEST_EMAIL}}"
|
||||
cf set-env notify-admin-${{inputs.environment}} NOTIFY_E2E_TEST_PASSWORD "${{secrets.NOTIFY_E2E_TEST_PASSWORD}}"
|
||||
cf set-env notify-api-${{inputs.environment}} E2E_BUCKET_NAME "${{secrets.E2E_BUCKET_NAME}}"
|
||||
|
||||
- name: Restage ${{matrix.app}}
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-${{ inputs.environment }}
|
||||
command: |
|
||||
cf restage --strategy rolling notify-${{matrix.app}}-${{inputs.environment}}
|
||||
- name: Restage ${{matrix.app}} egress
|
||||
uses: cloud-gov/cg-cli-tools@main
|
||||
with:
|
||||
cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
cf_org: gsa-tts-benefits-studio
|
||||
cf_space: notify-${{ inputs.environment }}-egress
|
||||
command: "cf restage --strategy rolling egress-proxy-notify-${{matrix.app}}-${{inputs.environment}}"
|
||||
87
.github/workflows/terraform-demo.yml
vendored
87
.github/workflows/terraform-demo.yml
vendored
@@ -1,87 +0,0 @@
|
||||
name: Run Terraform plan in demo
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ production ]
|
||||
paths: [ 'terraform/**' ]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: terraform/demo
|
||||
|
||||
jobs:
|
||||
terraform:
|
||||
name: Terraform plan
|
||||
runs-on: ubuntu-latest
|
||||
environment: demo
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform format
|
||||
id: format
|
||||
run: terraform fmt -check
|
||||
|
||||
- name: Terraform init
|
||||
id: init
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
|
||||
- name: Terraform validate
|
||||
id: validation
|
||||
run: terraform validate -no-color
|
||||
|
||||
- name: Terraform plan
|
||||
id: plan
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
|
||||
|
||||
- name: Read Terraform plan output file
|
||||
id: terraform_output
|
||||
uses: juliangruber/read-file-action@v1
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
path: ./terraform/demo/plan_output.txt
|
||||
|
||||
# inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
|
||||
- name: Update PR
|
||||
uses: actions/github-script@v7
|
||||
# we would like to update the PR even when a prior step failed
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
script: |
|
||||
const output = `Terraform Format and Style: ${{ steps.format.outcome }}
|
||||
Terraform Initialization: ${{ steps.init.outcome }}
|
||||
Terraform Validation: ${{ steps.validation.outcome }}
|
||||
Terraform Plan: ${{ steps.plan.outcome }}
|
||||
|
||||
<details><summary>Show Plan</summary>
|
||||
|
||||
\`\`\`\n
|
||||
${{ steps.terraform_output.outputs.content }}
|
||||
\`\`\`
|
||||
|
||||
</details>
|
||||
|
||||
*Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
|
||||
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: output
|
||||
})
|
||||
87
.github/workflows/terraform-production.yml
vendored
87
.github/workflows/terraform-production.yml
vendored
@@ -1,87 +0,0 @@
|
||||
name: Run Terraform plan in production
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ production ]
|
||||
paths: [ 'terraform/**' ]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: terraform/production
|
||||
|
||||
jobs:
|
||||
terraform:
|
||||
name: Terraform plan
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform format
|
||||
id: format
|
||||
run: terraform fmt -check
|
||||
|
||||
- name: Terraform init
|
||||
id: init
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
|
||||
- name: Terraform validate
|
||||
id: validation
|
||||
run: terraform validate -no-color
|
||||
|
||||
- name: Terraform plan
|
||||
id: plan
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
|
||||
|
||||
- name: Read Terraform plan output file
|
||||
id: terraform_output
|
||||
uses: juliangruber/read-file-action@v1
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
path: ./terraform/production/plan_output.txt
|
||||
|
||||
# inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
|
||||
- name: Update PR
|
||||
uses: actions/github-script@v7
|
||||
# we would like to update the PR even when a prior step failed
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
script: |
|
||||
const output = `Terraform Format and Style: ${{ steps.format.outcome }}
|
||||
Terraform Initialization: ${{ steps.init.outcome }}
|
||||
Terraform Validation: ${{ steps.validation.outcome }}
|
||||
Terraform Plan: ${{ steps.plan.outcome }}
|
||||
|
||||
<details><summary>Show Plan</summary>
|
||||
|
||||
\`\`\`\n
|
||||
${{ steps.terraform_output.outputs.content }}
|
||||
\`\`\`
|
||||
|
||||
</details>
|
||||
|
||||
*Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
|
||||
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: output
|
||||
})
|
||||
87
.github/workflows/terraform-staging.yml
vendored
87
.github/workflows/terraform-staging.yml
vendored
@@ -1,87 +0,0 @@
|
||||
name: Run Terraform plan in staging
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths: [ 'terraform/**' ]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: terraform/staging
|
||||
|
||||
jobs:
|
||||
terraform:
|
||||
name: Terraform plan
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Looks like we need to install Terraform ourselves now!
|
||||
# https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
with:
|
||||
terraform_version: "^1.7.5"
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Terraform format
|
||||
id: format
|
||||
run: terraform fmt -check
|
||||
|
||||
- name: Terraform init
|
||||
id: init
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
run: terraform init
|
||||
|
||||
- name: Terraform validate
|
||||
id: validation
|
||||
run: terraform validate -no-color
|
||||
|
||||
- name: Terraform plan
|
||||
id: plan
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
|
||||
TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
|
||||
TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
|
||||
run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
|
||||
|
||||
- name: Read Terraform plan output file
|
||||
id: terraform_output
|
||||
uses: juliangruber/read-file-action@v1
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
path: ./terraform/staging/plan_output.txt
|
||||
|
||||
# inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
|
||||
- name: Update PR
|
||||
uses: actions/github-script@v7
|
||||
# we would like to update the PR even when a prior step failed
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
script: |
|
||||
const output = `Terraform Format and Style: ${{ steps.format.outcome }}
|
||||
Terraform Initialization: ${{ steps.init.outcome }}
|
||||
Terraform Validation: ${{ steps.validation.outcome }}
|
||||
Terraform Plan: ${{ steps.plan.outcome }}
|
||||
|
||||
<details><summary>Show Plan</summary>
|
||||
|
||||
\`\`\`\n
|
||||
${{ steps.terraform_output.outputs.content }}
|
||||
\`\`\`
|
||||
|
||||
</details>
|
||||
|
||||
*Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
|
||||
|
||||
github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: output
|
||||
})
|
||||
21
.gitignore
vendored
21
.gitignore
vendored
@@ -4,7 +4,6 @@ queues.csv
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
.venv/
|
||||
venv/
|
||||
venv-freeze/
|
||||
|
||||
@@ -29,7 +28,6 @@ var/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
/cache
|
||||
requirements.txt
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
@@ -45,7 +43,6 @@ pip-delete-this-directory.txt
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage_cache
|
||||
.coverage.*
|
||||
.cache
|
||||
.pytest_cache
|
||||
@@ -59,7 +56,6 @@ test_results.xml
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
/logs/*
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
@@ -73,27 +69,10 @@ target/
|
||||
*.DS_Store
|
||||
environment.sh
|
||||
.envrc
|
||||
.env
|
||||
.env*
|
||||
varsfile
|
||||
|
||||
celerybeat-schedule
|
||||
celerybeat-schedule.db
|
||||
|
||||
# CloudFoundry
|
||||
.cf
|
||||
varsfile*
|
||||
.secret*
|
||||
|
||||
/scripts/run_my_tests.sh
|
||||
|
||||
# Terraform
|
||||
.terraform.lock.hcl
|
||||
**/.terraform/*
|
||||
secrets.auto.tfvars
|
||||
terraform.tfstate
|
||||
terraform.tfstate.backup
|
||||
|
||||
|
||||
# Pyenv
|
||||
.python-version
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.6.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- id: debug-statements
|
||||
- id: check-merge-conflict
|
||||
- id: check-toml
|
||||
- id: check-ast
|
||||
- id: fix-byte-order-marker
|
||||
- id: detect-aws-credentials
|
||||
args: [--allow-missing-credentials]
|
||||
- id: detect-private-key
|
||||
- id: mixed-line-ending
|
||||
- repo: https://github.com/Yelp/detect-secrets
|
||||
rev: v1.5.0
|
||||
hooks:
|
||||
- id: detect-secrets
|
||||
args: ['--baseline', '.ds.baseline']
|
||||
9
.profile
9
.profile
@@ -1,9 +0,0 @@
|
||||
##
|
||||
# Cloud Foundry app initialization script
|
||||
# https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile
|
||||
##
|
||||
|
||||
export http_proxy=$egress_proxy
|
||||
export https_proxy=$egress_proxy
|
||||
export NEW_RELIC_PROXY_HOST=$egress_proxy
|
||||
export no_proxy="apps.internal,s3-fips.us-gov-west-1.amazonaws.com"
|
||||
8
.pyup.yml
Normal file
8
.pyup.yml
Normal file
@@ -0,0 +1,8 @@
|
||||
# see https://pyup.io/docs/configuration/ for all available options
|
||||
|
||||
schedule: "every week on wednesday"
|
||||
|
||||
search: False
|
||||
requirements:
|
||||
- requirements-app.txt
|
||||
- requirements-dev.txt
|
||||
@@ -1,19 +0,0 @@
|
||||
# Welcome!
|
||||
|
||||
We're so glad you're thinking about contributing to a [open source project of the U.S. government](https://code.gov/)! If you're unsure about anything, just ask -- or submit the issue or pull request anyway. The worst that can happen is you'll be politely asked to change something. We love all friendly contributions.
|
||||
|
||||
We encourage you to read this project's CONTRIBUTING policy (you are here), its [LICENSE](LICENSE.md), and its [README](README.md).
|
||||
|
||||
## Policies
|
||||
|
||||
We want to ensure a welcoming environment for all of our projects. Our staff follow the [TTS Code of Conduct](https://18f.gsa.gov/code-of-conduct/) and all contributors should do the same.
|
||||
|
||||
This project is adhering to the [18F Open Source Policy](https://github.com/18f/open-source-policy).
|
||||
|
||||
As part of a U.S. government agency, the General Services Administration (GSA)’s Technology Transformation Services (TTS) takes seriously our responsibility to protect the public’s information, including financial and personal information, from unwarranted disclosure. For more information about security and vulnerability disclosure for our projects, please read our [18F Vulnerability Disclosure Policy](https://18f.gsa.gov/vulnerability-disclosure-policy/).
|
||||
|
||||
## Public domain
|
||||
|
||||
This project is in the public domain within the United States, and applicable copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
|
||||
|
||||
All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest.
|
||||
22
LICENSE
Normal file
22
LICENSE
Normal file
@@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Government Digital Service
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
53
LICENSE.md
53
LICENSE.md
@@ -1,53 +0,0 @@
|
||||
# License
|
||||
|
||||
## A few parts of this project are not in the public domain
|
||||
|
||||
### Copyright ownership of Works created by U.S. federal employees
|
||||
|
||||
This repository contains both the original work for [GOV.UK Notify](https://github.com/alphagov/notifications-api) (the Software, as defined in the MIT License) and modifications made by the General Services Administration (GSA). This repository combines the Software and these GSA modifications, which is a “joint work” of the Software and modifications combined into a single work.
|
||||
|
||||
As a work of the United States Government, the software modifications made by GSA are not subject to copyright within the United States. Additionally, GSA waives copyright and related rights in its software modifications worldwide through the [CC0 1.0 Universal Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/).
|
||||
|
||||
The Software remains subject to copyright under the MIT License. This consists of all work until [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) on June 1, 2022.
|
||||
|
||||
#### Full license text for the MIT licensed files:
|
||||
|
||||
```
|
||||
The MIT License (MIT)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
```
|
||||
|
||||
## The rest of this project is in the worldwide public domain
|
||||
|
||||
As a work of the United States government, this project is in the public domain within the United States.
|
||||
|
||||
Additionally, we waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
|
||||
|
||||
### CC0 1.0 Universal Summary
|
||||
|
||||
This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode).
|
||||
|
||||
#### No copyright
|
||||
|
||||
The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law.
|
||||
|
||||
You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.
|
||||
|
||||
#### Other information
|
||||
|
||||
In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights.
|
||||
|
||||
Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer.
|
||||
|
||||
## Contributions to this project
|
||||
|
||||
As stated in [CONTRIBUTING](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.
|
||||
|
||||
## Trademark
|
||||
|
||||
GSA owns, reserves, and retains all rights, title, and interest in all trademarks owned by GSA. The User does not acquire any right or interest of any kind in any GSA trademark because of their use of the Modified Version or any of the modifications made to the Work by GSA.
|
||||
295
Makefile
295
Makefile
@@ -5,165 +5,194 @@ DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
|
||||
APP_VERSION_FILE = app/version.py
|
||||
|
||||
GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
|
||||
GIT_COMMIT ?= $(shell git rev-parse HEAD 2> /dev/null || echo "")
|
||||
GIT_HOOKS_PATH ?= $(shell git config --global core.hooksPath || echo "")
|
||||
GIT_COMMIT ?= $(shell git rev-parse HEAD)
|
||||
|
||||
## DEVELOPMENT
|
||||
DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
|
||||
DOCKER_TTY ?= $(if ${JENKINS_HOME},,t)
|
||||
|
||||
## TODO this line should go under `make generate-version-file`
|
||||
## poetry self update
|
||||
BUILD_TAG ?= notifications-api-manual
|
||||
BUILD_NUMBER ?= 0
|
||||
DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
|
||||
BUILD_URL ?=
|
||||
|
||||
.PHONY: bootstrap
|
||||
bootstrap: ## Set up everything to run the app
|
||||
make generate-version-file
|
||||
poetry sync --no-root
|
||||
poetry run pre-commit install
|
||||
createdb notification_api || true
|
||||
createdb test_notification_api || true
|
||||
(poetry run flask db upgrade) || true
|
||||
DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
|
||||
|
||||
.PHONY: bootstrap-with-git-hooks
|
||||
bootstrap-with-git-hooks: ## Sets everything up and accounts for pre-existing git hooks
|
||||
make generate-version-file
|
||||
poetry sync --no-root
|
||||
git config --global --unset-all core.hooksPath
|
||||
poetry run pre-commit install
|
||||
git config --global core.hookspath "${GIT_HOOKS_PATH}"
|
||||
createdb notification_api || true
|
||||
createdb test_notification_api || true
|
||||
(poetry run flask db upgrade) || true
|
||||
CF_API ?= api.cloud.service.gov.uk
|
||||
CF_ORG ?= govuk-notify
|
||||
CF_SPACE ?= ${DEPLOY_ENV}
|
||||
CF_HOME ?= ${HOME}
|
||||
$(eval export CF_HOME)
|
||||
|
||||
.PHONY: bootstrap-with-docker
|
||||
bootstrap-with-docker: ## Build the image to run the app in Docker
|
||||
docker build -f docker/Dockerfile -t notifications-api .
|
||||
CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
|
||||
|
||||
.PHONY: run-procfile
|
||||
run-procfile:
|
||||
poetry run honcho start -f Procfile.dev
|
||||
|
||||
|
||||
|
||||
.PHONY: tada
|
||||
tada:
|
||||
poetry run isort .
|
||||
poetry run black .
|
||||
poetry run flake8 .
|
||||
|
||||
.PHONY: avg-complexity
|
||||
avg-complexity:
|
||||
echo "*** Shows average complexity in radon of all code ***"
|
||||
poetry run radon cc ./app -a -na
|
||||
|
||||
.PHONY: too-complex
|
||||
too-complex:
|
||||
echo "*** Shows code that got a rating of C, D or F in radon ***"
|
||||
poetry run radon cc ./app -a -nc
|
||||
|
||||
.PHONY: run-flask
|
||||
run-flask:
|
||||
poetry run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
|
||||
|
||||
.PHONY: run-celery
|
||||
run-celery: ## Run celery, TODO remove purge for staging/prod
|
||||
poetry run celery -A run_celery.notify_celery purge -f
|
||||
poetry run newrelic-admin run-program celery \
|
||||
-A run_celery.notify_celery worker \
|
||||
--pidfile="/tmp/celery.pid" \
|
||||
--loglevel=INFO \
|
||||
--pool=eventlet
|
||||
--concurrency=20
|
||||
|
||||
|
||||
.PHONY: dead-code
|
||||
dead-code: ## Use 60 to look for suspected dead code
|
||||
poetry run vulture ./app --min-confidence=100
|
||||
|
||||
.PHONY: run-celery-beat
|
||||
run-celery-beat: ## Run celery beat
|
||||
poetry run celery \
|
||||
-A run_celery.notify_celery beat \
|
||||
--loglevel=INFO
|
||||
|
||||
.PHONY: cloudgov-user-report
|
||||
cloudgov-user-report:
|
||||
@poetry run python -m terraform.ops.cloudgov_user_report
|
||||
NOTIFY_CREDENTIALS ?= ~/.notify-credentials
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
.PHONY: check-env-vars
|
||||
check-env-vars: ## Check mandatory environment variables
|
||||
$(if ${DEPLOY_ENV},,$(error Must specify DEPLOY_ENV))
|
||||
|
||||
.PHONY: preview
|
||||
preview: ## Set environment to preview
|
||||
$(eval export DEPLOY_ENV=preview)
|
||||
@true
|
||||
|
||||
.PHONY: staging
|
||||
staging: ## Set environment to staging
|
||||
$(eval export DEPLOY_ENV=staging)
|
||||
@true
|
||||
|
||||
.PHONY: production
|
||||
production: ## Set environment to production
|
||||
$(eval export DEPLOY_ENV=production)
|
||||
@true
|
||||
|
||||
.PHONY: generate-version-file
|
||||
generate-version-file: ## Generates the app version file
|
||||
@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
|
||||
@echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE}
|
||||
|
||||
.PHONY: build-paas-artifact
|
||||
build-paas-artifact: ## Build the deploy artifact for PaaS
|
||||
rm -rf target
|
||||
mkdir -p target
|
||||
zip -y -q -r -x@deploy-exclude.lst target/notifications-api.zip ./
|
||||
|
||||
.PHONY: upload-paas-artifact
|
||||
upload-paas-artifact: ## Upload the deploy artifact for PaaS
|
||||
$(if ${DEPLOY_BUILD_NUMBER},,$(error Must specify DEPLOY_BUILD_NUMBER))
|
||||
$(if ${JENKINS_S3_BUCKET},,$(error Must specify JENKINS_S3_BUCKET))
|
||||
aws s3 cp --region eu-west-1 --sse AES256 target/notifications-api.zip s3://${JENKINS_S3_BUCKET}/build/notifications-api/${DEPLOY_BUILD_NUMBER}.zip
|
||||
|
||||
.PHONY: test
|
||||
test: export NEW_RELIC_ENVIRONMENT=test
|
||||
test: ## Run tests and create coverage report
|
||||
poetry run black .
|
||||
poetry run flake8 .
|
||||
poetry run isort ./app ./tests
|
||||
poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10
|
||||
|
||||
|
||||
poetry run coverage report -m --fail-under=95
|
||||
poetry run coverage html -d .coverage_cache
|
||||
|
||||
.PHONY: test-debug
|
||||
test-debug:
|
||||
poetry run pytest --pdb -x
|
||||
|
||||
.PHONY: py-lock
|
||||
py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
|
||||
poetry sync --no-root
|
||||
poetry lock
|
||||
test: generate-version-file ## Run tests
|
||||
./scripts/run_tests.sh
|
||||
|
||||
.PHONY: freeze-requirements
|
||||
freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
|
||||
poetry export --output > requirements.txt
|
||||
rm -rf venv-freeze
|
||||
virtualenv -p python3 venv-freeze
|
||||
$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
|
||||
echo '# pyup: ignore file' > requirements.txt
|
||||
echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
|
||||
cat requirements-app.txt >> requirements.txt
|
||||
echo '' >> requirements.txt
|
||||
$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
|
||||
rm -rf venv-freeze
|
||||
|
||||
.PHONY: audit
|
||||
audit:
|
||||
poetry requirements > requirements.txt
|
||||
poetry requirements --dev > requirements_for_test.txt
|
||||
poetry run pip-audit -r requirements.txt --skip-editable
|
||||
poetry run pip-audit -r requirements_for_test.txt --skip-editable
|
||||
.PHONY: test-requirements
|
||||
test-requirements:
|
||||
@diff requirements-app.txt requirements.txt | grep '<' \
|
||||
&& { echo "requirements.txt doesn't match requirements-app.txt."; \
|
||||
echo "Run 'make freeze-requirements' to update."; exit 1; } \
|
||||
|| { echo "requirements.txt is up to date"; exit 0; }
|
||||
|
||||
.PHONY: static-scan
|
||||
static-scan:
|
||||
poetry run bandit -r app/
|
||||
.PHONY: prepare-docker-build-image
|
||||
prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
|
||||
docker build -f docker/Dockerfile \
|
||||
--build-arg HTTP_PROXY="${HTTP_PROXY}" \
|
||||
--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
|
||||
--build-arg NO_PROXY="${NO_PROXY}" \
|
||||
-t ${DOCKER_BUILDER_IMAGE_NAME} \
|
||||
.
|
||||
|
||||
.PHONY: build-with-docker
|
||||
build-with-docker: ; ## don't do anything
|
||||
|
||||
.PHONY: test-with-docker
|
||||
test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
|
||||
@docker run -i${DOCKER_TTY} --rm \
|
||||
--name "${DOCKER_CONTAINER_PREFIX}-test" \
|
||||
--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
|
||||
-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
|
||||
-e GIT_COMMIT=${GIT_COMMIT} \
|
||||
-e BUILD_NUMBER=${BUILD_NUMBER} \
|
||||
-e BUILD_URL=${BUILD_URL} \
|
||||
-e http_proxy="${HTTP_PROXY}" \
|
||||
-e HTTP_PROXY="${HTTP_PROXY}" \
|
||||
-e https_proxy="${HTTPS_PROXY}" \
|
||||
-e HTTPS_PROXY="${HTTPS_PROXY}" \
|
||||
-e NO_PROXY="${NO_PROXY}" \
|
||||
${DOCKER_BUILDER_IMAGE_NAME} \
|
||||
make test
|
||||
|
||||
.PHONY: create-docker-test-db
|
||||
create-docker-test-db: ## Start the test database in a Docker container
|
||||
docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
|
||||
@docker run -d \
|
||||
--name "${DOCKER_CONTAINER_PREFIX}-db" \
|
||||
-e POSTGRES_PASSWORD="postgres" \
|
||||
-e POSTGRES_DB=test_notification_api \
|
||||
postgres:9.5
|
||||
sleep 3
|
||||
|
||||
.PHONY: clean-docker-containers
|
||||
clean-docker-containers: ## Clean up any remaining docker containers
|
||||
docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
|
||||
rm -rf node_modules cache target venv .coverage build tests/.cache
|
||||
|
||||
.PHONY: cf-login
|
||||
cf-login: ## Log in to Cloud Foundry
|
||||
$(if ${CF_USERNAME},,$(error Must specify CF_USERNAME))
|
||||
$(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD))
|
||||
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
||||
@echo "Logging in to Cloud Foundry on ${CF_API}"
|
||||
@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
|
||||
|
||||
.PHONY: generate-manifest
|
||||
generate-manifest:
|
||||
$(if ${CF_APP},,$(error Must specify CF_APP))
|
||||
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
||||
$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
|
||||
$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
|
||||
|
||||
@jinja2 --strict manifest.yml.j2 \
|
||||
-D environment=${CF_SPACE} \
|
||||
-D CF_APP=${CF_APP} \
|
||||
--format=yaml \
|
||||
<(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
|
||||
|
||||
.PHONY: cf-deploy
|
||||
cf-deploy: ## Deploys the app to Cloud Foundry
|
||||
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
||||
$(if ${CF_APP},,$(error Must specify CF_APP))
|
||||
cf target -o ${CF_ORG} -s ${CF_SPACE}
|
||||
@cf app --guid ${CF_APP} || exit 1
|
||||
|
||||
# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
|
||||
cf v3-cancel-zdt-push ${CF_APP} || true
|
||||
|
||||
cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
|
||||
CF_STARTUP_TIMEOUT=10 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work
|
||||
|
||||
|
||||
## DEPLOYMENT
|
||||
.PHONY: cf-deploy-api-db-migration
|
||||
cf-deploy-api-db-migration:
|
||||
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
|
||||
cf target -o ${CF_ORG} -s ${CF_SPACE}
|
||||
cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
|
||||
cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
|
||||
|
||||
# .PHONY: cf-deploy-failwhale
|
||||
# cf-deploy-failwhale:
|
||||
# $(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
|
||||
# cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
|
||||
.PHONY: cf-check-api-db-migration-task
|
||||
cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
|
||||
@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"
|
||||
|
||||
# .PHONY: enable-failwhale
|
||||
# enable-failwhale: ## Enable the failwhale app and disable api
|
||||
# $(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`))
|
||||
# # make sure failwhale is running first
|
||||
# cf start notify-api-failwhale
|
||||
.PHONY: cf-rollback
|
||||
cf-rollback: ## Rollbacks the app to the previous release
|
||||
$(if ${CF_APP},,$(error Must specify CF_APP))
|
||||
cf v3-cancel-zdt-push ${CF_APP}
|
||||
|
||||
# cf map-route notify-api-failwhale ${DNS_NAME} --hostname api
|
||||
# cf unmap-route notify-api ${DNS_NAME} --hostname api
|
||||
# @echo "Failwhale is enabled"
|
||||
.PHONY: cf-push
|
||||
cf-push:
|
||||
$(if ${CF_APP},,$(error Must specify CF_APP))
|
||||
cf target -o ${CF_ORG} -s ${CF_SPACE}
|
||||
cf push ${CF_APP} -f <(make -s generate-manifest)
|
||||
|
||||
# .PHONY: disable-failwhale
|
||||
# disable-failwhale: ## Disable the failwhale app and enable api
|
||||
# $(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`))
|
||||
|
||||
# cf map-route notify-api ${DNS_NAME} --hostname api
|
||||
# cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
|
||||
# cf stop notify-api-failwhale
|
||||
# @echo "Failwhale is disabled"
|
||||
|
||||
.PHONY: test-single
|
||||
test-single: export NEW_RELIC_ENVIRONMENT=test
|
||||
test-single: ## Run a single test file
|
||||
poetry run pytest -s $(TEST_FILE)
|
||||
.PHONY: check-if-migrations-to-run
|
||||
check-if-migrations-to-run:
|
||||
@echo $(shell python3 scripts/check_if_new_migration.py)
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
web: make run-flask
|
||||
worker: make run-celery
|
||||
scheduler: make run-celery-beat
|
||||
606
README.md
606
README.md
@@ -1,562 +1,158 @@
|
||||
|
||||
# GOV.UK Notify API
|
||||
|
||||
# Notify.gov API
|
||||
Contains:
|
||||
- the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation)
|
||||
- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to)
|
||||
- asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc
|
||||
|
||||
This project is the core of [Notify.gov](https://notify-demo.app.cloud.gov).
|
||||
It's cloned from the brilliant work of the team at
|
||||
[GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
|
||||
## Setting Up
|
||||
|
||||
This repo contains:
|
||||
### Python version
|
||||
|
||||
- A public-facing REST API for Notify.gov, which teams can integrate with using
|
||||
[API clients built by UK](https://www.notifications.service.gov.uk/documentation).
|
||||
- An internal-only REST API built using Flask to manage services, users,
|
||||
templates, etc., which the
|
||||
[Notify.gov Admin UI](http://github.com/18F/notifications-admin) talks to.
|
||||
- Asynchronous workers built using Celery to put things on queues and read them
|
||||
off to be processed, sent to providers, updated, etc.
|
||||
At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.
|
||||
|
||||
Our other repositories are:
|
||||
### AWS credentials
|
||||
|
||||
- [us-notify-compliance](https://github.com/GSA/us-notify-compliance/)
|
||||
- [notify-python-demo](https://github.com/GSA/notify-python-demo)
|
||||
To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.
|
||||
|
||||
## Before You Start
|
||||
Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
|
||||
|
||||
You will need the following items:
|
||||
|
||||
- An active cloud.gov account with the correct permissions - speak with your
|
||||
onboarding buddy for help with
|
||||
[setting up an account](https://cloud.gov/sign-up/) (requires a `.mil`,
|
||||
`.gov`, or `.fed.us` email address) and getting access to the
|
||||
`notify-local-dev` and `notify-staging` spaces.
|
||||
- Admin priviliges and SSH access on your machine; you may need to work with
|
||||
your organization's IT support staff if you're not sure or don't currently
|
||||
have this access.
|
||||
|
||||
## Local Environment Setup
|
||||
|
||||
This project currently works with these major versions of the following main
|
||||
components:
|
||||
|
||||
- Python 3.13.x
|
||||
- PostgreSQL 15.x (version 12.x is used in the hosted environments)
|
||||
|
||||
These instructions will walk you through how to set your machine up with all of
|
||||
the required tools for this project.
|
||||
|
||||
### Project Pre-Requisite Setup
|
||||
|
||||
On MacOS, using [Homebrew](https://brew.sh/) for package management is highly
|
||||
recommended. This helps avoid some known installation issues. Start by following
|
||||
the installation instructions on the Homebrew homepage.
|
||||
|
||||
**Note:** You will also need Xcode or the Xcode Command Line Tools installed. The
|
||||
quickest way to do this is by installing the command line tools in the shell:
|
||||
|
||||
```sh
|
||||
xcode-select –-install
|
||||
```
|
||||
|
||||
#### Homebrew Setup
|
||||
|
||||
If this is your first time installing Homebrew on your machine, you may need to
|
||||
add its binaries to your system's `$PATH` environment variable so that you can
|
||||
use the `brew` command. Try running `brew help` to see if Homebrew is
|
||||
recognized and runs properly. If that fails, then you'll need to add a
|
||||
configuration line to wherever your `$PATH` environment variable is set.
|
||||
|
||||
Your system `$PATH` environment variable is likely set in one of these
|
||||
locations:
|
||||
|
||||
For BASH shells:
|
||||
|
||||
- `~/.bashrc`
|
||||
- `~/.bash_profile`
|
||||
- `~/.profile`
|
||||
|
||||
For ZSH shells:
|
||||
|
||||
- `~/.zshrc`
|
||||
- `~/.zprofile`
|
||||
|
||||
There may be different files that you need to modify for other shell
|
||||
environments.
|
||||
|
||||
Which file you need to modify depends on whether or not you are running an
|
||||
interactive shell or a login shell
|
||||
(see [this Stack Overflow post](https://stackoverflow.com/questions/18186929/what-are-the-differences-between-a-login-shell-and-interactive-shell)
|
||||
for an explanation of the differences). If you're still not sure, please ask
|
||||
the team for help!
|
||||
|
||||
Once you determine which file you'll need to modify, add these lines before any
|
||||
lines that add or modify the `$PATH` environment variable; near or at the top
|
||||
of the file is appropriate:
|
||||
|
||||
```sh
|
||||
# Homebrew setup
|
||||
eval "$(/opt/homebrew/bin/brew shellenv)"
|
||||
```
|
||||
|
||||
This will make sure Homebrew gets setup correctly. Once you make these changes,
|
||||
either start a new shell session or source the file
|
||||
(`source ~/.FILE-YOU-MODIFIED`) you modified to have your system recognize the
|
||||
changes.
|
||||
|
||||
Verify that Homebrew is now working by trying to run `brew help` again.
|
||||
|
||||
### System-Level Package Installation
|
||||
|
||||
There are several packages you will need to install for your system in order to
|
||||
get the app running (and these are good to have in general for any software
|
||||
development).
|
||||
|
||||
Start off with these packages since they're quick and don't require additional
|
||||
configuration after installation to get working out of the box:
|
||||
|
||||
- [jq](https://stedolan.github.io/jq/) - for working with JSON in the command
|
||||
line
|
||||
- [git](https://git-scm.com/) - for version control management
|
||||
- [tfenv](https://github.com/tfutils/tfenv) - for managing
|
||||
[Terraform](https://www.terraform.io/) installations
|
||||
- [cf-cli@8](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html) - for
|
||||
working with a Cloud Foundry platform (e.g., cloud.gov)
|
||||
- [redis](https://redis.io/) - required as the backend for the API's
|
||||
asynchronous job processing
|
||||
- [vim](https://www.vim.org/) - for editing files more easily in the command
|
||||
line
|
||||
- [wget](https://www.gnu.org/software/wget/) - for retrieving files in the
|
||||
command line
|
||||
|
||||
You can install them by running the following:
|
||||
|
||||
```sh
|
||||
brew install jq git tfenv cloudfoundry/tap/cf-cli@8 redis vim wget
|
||||
```
|
||||
|
||||
#### Terraform Installation
|
||||
|
||||
As a part of the installation above, you just installed `tfenv` to manage
|
||||
Terraform installations. This is great, but you still need to install Terraform
|
||||
itself, which can be done with this command:
|
||||
|
||||
```sh
|
||||
tfenv install "latest:^1.7"
|
||||
tfenv use 1.7.x # x = the patch version installed
|
||||
```
|
||||
|
||||
#### Python Installation
|
||||
|
||||
Now we're going to install a tool to help us manage Python versions and
|
||||
virtual environments on our system. First, we'll install
|
||||
[pyenv](https://github.com/pyenv/pyenv) and one of its plugins,
|
||||
[pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv), with Homebrew:
|
||||
|
||||
```sh
|
||||
brew install pyenv pyenv-virtualenv
|
||||
```
|
||||
|
||||
When these finish installing, you'll need to make another adjustment in the
|
||||
file that you adjusted for your `$PATH` environment variable and Homebrew's
|
||||
setup. Open the file, and add these lines to it:
|
||||
### Virtualenv
|
||||
|
||||
```
|
||||
# pyenv setup
|
||||
export PYENV_ROOT="$HOME/.pyenv"
|
||||
command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
|
||||
eval "$(pyenv init -)"
|
||||
eval "$(pyenv virtualenv-init -)"
|
||||
mkvirtualenv -p /usr/local/bin/python3 notifications-api
|
||||
```
|
||||
|
||||
Once again, start a new shell session or source the file in your current shell
|
||||
session to make the changes take effect.
|
||||
### `environment.sh`
|
||||
|
||||
Now we're ready to install the Python version we need with `pyenv`, like so:
|
||||
Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
|
||||
|
||||
```sh
|
||||
pyenv install 3.13
|
||||
```
|
||||
|
||||
This will install the latest version of Python 3.13.
|
||||
|
||||
_NOTE: This project currently runs on Python 3.13.x._
|
||||
|
||||
#### Python Dependency Installation
|
||||
|
||||
Lastly, we need to install the tool we use to manage Python dependencies within
|
||||
the project, which is [poetry](https://python-poetry.org/).
|
||||
|
||||
Visit the
|
||||
[official installer instructions page](https://python-poetry.org/docs/#installing-with-the-official-installer)
|
||||
and follow the steps to install Poetry directly with the script.
|
||||
|
||||
This will ensure `poetry` doesn't conflict with any project virtual environments
|
||||
and can update itself properly.
|
||||
|
||||
#### PostgreSQL installation
|
||||
|
||||
We now need to install a database - this project uses PostgreSQL, and Homebrew
|
||||
requires a version number to be included with it when installing it:
|
||||
|
||||
```sh
|
||||
brew install postgresql@15
|
||||
```
|
||||
|
||||
You'll now need to modify (or create, if it doesn't already exist) the `$PATH`
|
||||
environment variable to include the PostgreSQL binaries. Open the file you have
|
||||
worked with before to adjust your shell environment with the previous steps and
|
||||
do one of the following:
|
||||
|
||||
If you already have a line that modifies the `$PATH` environment variable, just
|
||||
add this path into the existing string:
|
||||
Create a local environment.sh file containing the following:
|
||||
|
||||
```
|
||||
/opt/homebrew/opt/postgresql@15/bin
|
||||
echo "
|
||||
export NOTIFY_ENVIRONMENT='development'
|
||||
|
||||
export MMG_API_KEY='MMG_API_KEY'
|
||||
export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
|
||||
export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
|
||||
|
||||
export FLASK_APP=application.py
|
||||
export FLASK_DEBUG=1
|
||||
export WERKZEUG_DEBUG_PIN=off
|
||||
"> environment.sh
|
||||
```
|
||||
|
||||
If you don't have a line for your `$PATH` environment variable, add it in like
|
||||
this, which will include the PostgreSQL binaries:
|
||||
NOTES:
|
||||
|
||||
* Replace the placeholder key and prefix values as appropriate
|
||||
* The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
|
||||
* The unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.
|
||||
|
||||
### Postgres
|
||||
|
||||
Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
|
||||
|
||||
Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.
|
||||
|
||||
### Redis
|
||||
|
||||
To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
|
||||
|
||||
REDIS_ENABLED = True
|
||||
|
||||
|
||||
## To run the application
|
||||
|
||||
First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
|
||||
|
||||
You need to run the api application and a local celery instance.
|
||||
|
||||
There are two run scripts for running all the necessary parts.
|
||||
|
||||
```
|
||||
export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
|
||||
scripts/run_app.sh
|
||||
```
|
||||
|
||||
_NOTE: This project currently works with PostgreSQL version 15.x; version 12.x is currently used in our hosted environments._
|
||||
|
||||
_NOTE: If you have a pre-existing instance of PSQL installed because of another product like PGAdmin, your database configuration may differ from the instructions above, which uses Homebrew to install and configure PostgreSQL. If this is the case for you, you may have to either account for slightly different user permissions with the database, or uninstall PGAdmin and/or PostgreSQL itself, and reinstall it with Homebrew to follow the steps above._
|
||||
|
||||
_NOTE: You don't want to overwrite your existing `$PATH` environment variable! Hence the reason why it is included on the end like this; paths are separated by a colon._
|
||||
|
||||
#### Starting PostgreSQL and Redis
|
||||
|
||||
With both PostgreSQL and Redis installed, you now need to start the services.
|
||||
Run this command so that they're available at all times going forward on your
|
||||
machine:
|
||||
|
||||
```sh
|
||||
brew services start postgresql@15
|
||||
brew services start redis
|
||||
```
|
||||
scripts/run_celery.sh
|
||||
```
|
||||
|
||||
If they're already running, you can run this command instead to make sure the
|
||||
latest updates are applied to both services:
|
||||
|
||||
```sh
|
||||
brew services restart postgresql@15
|
||||
brew services restart redis
|
||||
```
|
||||
|
||||
### First-Time Project Setup
|
||||
|
||||
Once all of pre-requisites for the project are installed and you have a
|
||||
cloud.gov account, you can now set up the API project and get things running
|
||||
locally!
|
||||
|
||||
First, clone the repository in the directory of your choosing on your machine:
|
||||
|
||||
```sh
|
||||
git clone git@github.com:GSA/notifications-api.git
|
||||
```
|
||||
|
||||
Now go into the project directory (`notifications-api` by default), create a
|
||||
virtual environment, and set the local Python version to point to the virtual
|
||||
environment (assumes version Python `3.12.9` is what is installed on your
|
||||
machine):
|
||||
|
||||
```sh
|
||||
cd notifications-api
|
||||
pyenv virtualenv 3.12.9 notify-api
|
||||
pyenv local notify-api
|
||||
```
|
||||
|
||||
_NOTE: If you're not sure which version of Python was installed with `pyenv`, you can check by running `pyenv versions` and it'll list everything available currently._
|
||||
|
||||
Now [log into cloud.gov](https://cloud.gov/docs/getting-started/setup/#set-up-the-command-line)
|
||||
in the command line by using this command:
|
||||
|
||||
```sh
|
||||
cf login -a api.fr.cloud.gov --sso
|
||||
```
|
||||
|
||||
If you are offered a choice of orgs, select `gsa-tts-benefits-studio`.
|
||||
For the space, choose `notify-local-dev` to start with (assuming you are
|
||||
setting up local development).
|
||||
|
||||
_REMINDER: Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov_
|
||||
|
||||
Now run the development Terraform setup by navigating to the development
|
||||
folder and running the script in it:
|
||||
|
||||
```sh
|
||||
cd terraform/development
|
||||
./run.sh
|
||||
```
|
||||
|
||||
If this runs correctly, Terraform will ask you if you want to create some
|
||||
resources. Answer `yes`.
|
||||
|
||||
The script will also create a local `.env` file for you in the project's
|
||||
root directory, which will include a handful of project-specific environment
|
||||
variables.
|
||||
|
||||
Lastly, if you didn't already start PostgreSQL and Redis above, be sure to do
|
||||
so now:
|
||||
|
||||
```sh
|
||||
brew services start postgresql@15
|
||||
brew services start redis
|
||||
```
|
||||
|
||||
#### Upgrading Python in existing projects
|
||||
|
||||
If you're upgrading an existing project to a newer version of Python, you can
|
||||
follow these steps to get yourself up-to-date.
|
||||
|
||||
First, use `pyenv` to install the newer version of Python you'd like to use;
|
||||
we'll use `3.13` in our example here since we recently upgraded to this version:
|
||||
|
||||
```sh
|
||||
pyenv install 3.13
|
||||
```
|
||||
|
||||
Next, delete the virtual environment you previously had set up. If you followed
|
||||
the instructions above with the first-time set up, you can do this with `pyenv`:
|
||||
|
||||
```sh
|
||||
pyenv virtualenv-delete notify-api
|
||||
```
|
||||
|
||||
Now, make sure you are in your project directory and recreate the same virtual
|
||||
environment with the newer version of Python you just installed:
|
||||
|
||||
```sh
|
||||
cd notifications-api
|
||||
pyenv virtualenv 3.12.9 notify-api
|
||||
pyenv local notify-api
|
||||
```
|
||||
|
||||
At this point, proceed with the rest of the instructions here in the README and
|
||||
you'll be set with an upgraded version of Python.
|
||||
|
||||
_NOTE: If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
|
||||
|
||||
#### Poetry upgrades
|
||||
|
||||
If you are doing a new project setup, then after you install poetry you need to install the export plugin
|
||||
|
||||
```sh
|
||||
poetry self add poetry-plugin-export
|
||||
```
|
||||
|
||||
If you are upgrading from poetry 1.8.5, you need to do this:
|
||||
|
||||
```sh
|
||||
curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
|
||||
poetry self add poetry-export-plugin
|
||||
```
|
||||
|
||||
### Final environment setup
|
||||
|
||||
There's one final thing to adjust in the newly created `.env` file. This
|
||||
project has support for end-to-end (E2E) tests and has some additional checks
|
||||
for the presence of an E2E test user so that it can be authenticated properly.
|
||||
|
||||
In the `.env` file, you should see this section:
|
||||
Optionally you can also run this script to run the scheduled tasks:
|
||||
|
||||
```
|
||||
#############################################################
|
||||
|
||||
# E2E Testing
|
||||
|
||||
NOTIFY_E2E_TEST_EMAIL=example@fake.gov
|
||||
NOTIFY_E2E_TEST_PASSWORD="don't write secrets to the sample file"
|
||||
scripts/run_celery_beat.sh
|
||||
```
|
||||
|
||||
You can leave the email address alone or change it to something else to your
|
||||
liking.
|
||||
|
||||
**You should absolutely change the `NOTIFY_E2E_TEST_PASSWORD` environment
|
||||
variable to something else, preferably a lengthy passphrase.**
|
||||
## To test the application
|
||||
|
||||
With those two environment variable set, the database migrations will run
|
||||
properly and an E2E test user will be ready to go for use in the admin project.
|
||||
First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
|
||||
|
||||
_Note: Whatever you set these two environment variables to, you'll need to
|
||||
match their values on the admin side. Please see the admin README and
|
||||
documentation for more details._
|
||||
Then simply run
|
||||
|
||||
## Running the Project and Routine Maintenance
|
||||
|
||||
The first time you run the project you'll need to run the project setup from the
|
||||
root project directory:
|
||||
|
||||
```sh
|
||||
make bootstrap
|
||||
```
|
||||
make test
|
||||
```
|
||||
|
||||
This command is handled by the `Makefile` file in the root project directory, as
|
||||
are a few others.
|
||||
That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
|
||||
[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
|
||||
|
||||
_NOTE: You'll want to occasionally run `make bootstrap` to keep your project up-to-date, especially when there are dependency updates._
|
||||
|
||||
Now you can run the web server and background workers for asynchronous jobs:
|
||||
## To update application dependencies
|
||||
|
||||
```sh
|
||||
make run-procfile
|
||||
`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
|
||||
versions of all nested dependencies. If `requirements-app.txt` has been changed (or
|
||||
we want to update the unpinned nested dependencies) `requirements.txt` should be
|
||||
regenerated with
|
||||
|
||||
```
|
||||
make freeze-requirements
|
||||
```
|
||||
|
||||
If it runs correctly, you will be able to visit http://127.0.0.1:6011/ and see
|
||||
JSON from the API in your web browser.
|
||||
`requirements.txt` should be committed alongside `requirements-app.txt` changes.
|
||||
|
||||
This will run all of the services within the same shell session. If you need to
|
||||
run them separately to help with debugging or tracing logs, you can do so by
|
||||
opening three sepearate shell sessions and running one of these commands in each
|
||||
one separately:
|
||||
|
||||
- `make run-celery` - Handles the asynchronous jobs
|
||||
- `make run-celery-beat` - Handles the scheduling of asynchronous jobs
|
||||
- `make run-flask` - Runs the web server
|
||||
## To run one off tasks
|
||||
|
||||
## Python Dependency Management
|
||||
Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
|
||||
care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
|
||||
example, to purge all dynamically generated functional test data, do the following:
|
||||
|
||||
We're using [`Poetry`](https://python-poetry.org/) for managing our Python
|
||||
dependencies and local virtual environments.
|
||||
|
||||
This project has two key dependency files that must be managed together:
|
||||
|
||||
- `pyproject.toml` - Contains the dependency specifications
|
||||
- `poetry.lock` - Contains the exact versions of all dependencies (including transitive ones)
|
||||
|
||||
### Managing Dependencies
|
||||
|
||||
There are two approaches for updating dependencies:
|
||||
|
||||
#### 1. Manual manipulation of `pyproject.toml`
|
||||
|
||||
If you manually edit the `pyproject.toml` file, you should use the `make py-lock` command to sync the `poetry.lock` file. This will
|
||||
ensure that you don't inadvertently bring in other transitive dependency updates
|
||||
that have not been fully tested with the project yet.
|
||||
|
||||
#### 2. Using Poetry to update dependencies (recommended)
|
||||
|
||||
If you're updating a dependency to a newer (or the latest) version,
|
||||
let Poetry handle it by running:
|
||||
|
||||
```sh
|
||||
poetry update <dependency> [<dependency>...]
|
||||
Locally
|
||||
```
|
||||
flask command purge_functional_test_data -u <functional tests user name prefix>
|
||||
```
|
||||
|
||||
You can specify more than one dependency together. With this command, Poetry
|
||||
will do the following for you:
|
||||
On the server
|
||||
```
|
||||
cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
|
||||
```
|
||||
|
||||
- Find the latest compatible version(s) of the specified dependency/dependencies
|
||||
- Install the new versions
|
||||
- Update and sync the `poetry.lock` file
|
||||
All commands and command options have a --help command if you need more information.
|
||||
|
||||
**Important:** In either situation, once you are finished and have verified the dependency
|
||||
changes are working, you must commit both the `pyproject.toml` and
|
||||
`poetry.lock` files together.
|
||||
|
||||
## Known Installation Issues
|
||||
## To create a new worker app
|
||||
|
||||
### Python Installation Errors
|
||||
You need to:
|
||||
1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
|
||||
1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
|
||||
1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
|
||||
1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
|
||||
|
||||
On M1 Macs, if you get a `fatal error: 'Python.h' file not found` message, try a
|
||||
different method of installing Python. The recommended approach is to use
|
||||
[`pyenv`](https://github.com/pyenv/pyenv), as noted above in the installation
|
||||
instructions.
|
||||
**Important:**
|
||||
|
||||
If you're using PyCharm for Python development, we've noticed some quirkiness
|
||||
with the IDE and the interaction between Poetry and virtual environment
|
||||
management that could cause a variety of problems to come up during project
|
||||
setup and dependency management. Other tools, such as Visual Studio Code, have
|
||||
proven to be a smoother experience for folks.
|
||||
Before pushing the deployment change on jenkins, read below about the first time deployment.
|
||||
|
||||
### PostgreSQL Installation Errors
|
||||
### First time deployment of your new worker
|
||||
|
||||
A direct installation of PostgreSQL will not put the `createdb` command on your
|
||||
`$PATH`. It can be added there in your shell startup script, or a
|
||||
Homebrew-managed installation of PostgreSQL will take care of it. See the
|
||||
instructions above for more details.
|
||||
Our deployment flow requires that the app is present in order to proceed with the deployment.
|
||||
|
||||
## Documentation
|
||||
This means that the first deployment of your app must happen manually.
|
||||
|
||||
- [Infrastructure overview](./docs/all.md#infrastructure-overview)
|
||||
- [GitHub Repositories](./docs/all.md#github-repositories)
|
||||
- [Terraform](./docs/all.md#terraform)
|
||||
- [AWS](./docs/all.md#aws)
|
||||
- [New Relic](./docs/all.md#new-relic)
|
||||
- [Onboarding](./docs/all.md#onboarding)
|
||||
- [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure)
|
||||
- [Using the logs](./docs/all.md#using-the-logs)
|
||||
- [`git` hooks](./docs/all.md#git-hooks)
|
||||
- [detect-secrets pre-commit plugin](./docs/all.md#detect-secrets-pre-commit-plugin)
|
||||
- [Testing](./docs/all.md#testing)
|
||||
- [CI testing](./docs/all.md#ci-testing)
|
||||
- [Manual testing](./docs/all.md#manual-testing)
|
||||
- [To run a local OWASP scan](./docs/all.md#to-run-a-local-owasp-scan)
|
||||
- [End-to-end testing](./docs/all.md#end-to-end-testing)
|
||||
- [Deploying](./docs/all.md#deploying)
|
||||
- [Egress Proxy](./docs/all.md#egress-proxy)
|
||||
- [Managing environment variables](./docs/all.md#managing-environment-variables)
|
||||
- [Managing application initialization](./docs/all.md#managing-application-initialization)
|
||||
- [Sandbox environment](./docs/all.md#sandbox-environment)
|
||||
- [Database management](./docs/all.md#database-management)
|
||||
- [Initial state](./docs/all.md#initial-state)
|
||||
- [Data Model Diagram](./docs/all.md#data-model-diagram)
|
||||
- [Migrations](./docs/all.md#migrations)
|
||||
- [Purging user data](./docs/all.md#purging-user-data)
|
||||
- [One-off tasks](./docs/all.md#one-off-tasks)
|
||||
- [Test Loading Commands](./docs/all.md#commands-for-test-loading-the-local-dev-database)
|
||||
- [How messages are queued and sent](./docs/all.md#how-messages-are-queued-and-sent)
|
||||
- [Writing public APIs](./docs/all.md#writing-public-apis)
|
||||
- [Overview](./docs/all.md#overview)
|
||||
- [Documenting APIs](./docs/all.md#documenting-apis)
|
||||
- [New APIs](./docs/all.md#new-apis)
|
||||
- [API Usage](./docs/all.md#api-usage)
|
||||
- [Connecting to the API](./docs/all.md#connecting-to-the-api)
|
||||
- [Postman Documentation](./docs/all.md#postman-documentation)
|
||||
- [Using OpenAPI documentation](./docs/all.md#using-openapi-documentation)
|
||||
- [Queues and tasks](./docs/all.md#queues-and-tasks)
|
||||
- [Priority queue](./docs/all.md#priority-queue)
|
||||
- [Celery scheduled tasks](./docs/all.md#celery-scheduled-tasks)
|
||||
- [Notify.gov](./docs/all.md#notifygov)
|
||||
- [System Description](./docs/all.md#system-description)
|
||||
- [Pull Requests](.docs/all.md#pull-requests)
|
||||
- [Getting Started](.docs/all.md#getting-started)
|
||||
- [Description](.docs/all.md#description)
|
||||
- [TODO (optional)](<.docs/all.md#todo-(optional)>)
|
||||
- [Security Considerations](.docs/all.md#security-considerations)
|
||||
- [Code Reviews](.docs/all.md#code-reviews)
|
||||
- [For the reviewer](.docs/all.md#for-the-reviewer)
|
||||
- [For the author](.docs/all.md#for-the-author)
|
||||
- [Run Book](./docs/all.md#run-book)
|
||||
- [Alerts, Notifications, Monitoring](./docs/all.md#-alerts-notifications-monitoring)
|
||||
- [Restaging Apps](./docs/all.md#-restaging-apps)
|
||||
- [Deploying to Production](./docs/all.md#-deploying-to-production)
|
||||
- [Smoke-testing the App](./docs/all.md#-smoke-testing-the-app)
|
||||
- [Configuration Management](./docs/all.md#-configuration-management)
|
||||
- [DNS and Domain Changes](./docs/all.md#-dns-and-domain-changes)
|
||||
- [Exporting daily scan results for compliance monitoring](./docs/all.md#exporting-daily-scan-results-for-compliance-monitoring)
|
||||
- [Reviewing daily scan results for compliance](./docs/all.md#reviewing-daily-scan-results-for-compliance)
|
||||
- [Rotating environment variable secrets](./docs/all.md#rotating-environment-variable-secrets)
|
||||
- [Known Gotchas](./docs/all.md#-known-gotchas)
|
||||
- [User Account Management](./docs/all.md#-user-account-management)
|
||||
- [SMS Phone Number Management](./docs/all.md#-sms-phone-number-management)
|
||||
- [Data Storage Policies \& Procedures](./docs/all.md#data-storage-policies--procedures)
|
||||
- [Potential PII Locations](./docs/all.md#potential-pii-locations)
|
||||
- [Data Retention Policy](./docs/all.md#data-retention-policy)
|
||||
To do this:
|
||||
|
||||
## License && public domain
|
||||
1. Ensure your code is backwards compatible
|
||||
1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
|
||||
|
||||
Work through
|
||||
[commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5)
|
||||
is licensed by the UK government under the MIT license. Work after that commit
|
||||
is in the worldwide public domain. See [LICENSE.md](./LICENSE.md) for more
|
||||
information.
|
||||
|
||||
## Contributing
|
||||
|
||||
As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this
|
||||
project will be released under the CC0 dedication. By submitting a pull request,
|
||||
you are agreeing to comply with this waiver of copyright interest.
|
||||
Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.
|
||||
|
||||
451
app/__init__.py
451
app/__init__.py
@@ -1,213 +1,137 @@
|
||||
import logging as real_logging
|
||||
import os
|
||||
import secrets
|
||||
import random
|
||||
import string
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
|
||||
from celery import Celery, Task, current_task
|
||||
from flask import current_app, g, has_request_context, jsonify, make_response, request
|
||||
from flask.ctx import has_app_context
|
||||
from flask_migrate import Migrate
|
||||
from flask_socketio import SocketIO
|
||||
from flask import _request_ctx_stack, request, g, jsonify, make_response
|
||||
from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
|
||||
from sqlalchemy import event
|
||||
from flask_marshmallow import Marshmallow
|
||||
from flask_migrate import Migrate
|
||||
from time import monotonic
|
||||
from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
|
||||
from notifications_utils.clients.statsd.statsd_client import StatsdClient
|
||||
from notifications_utils.clients.redis.redis_client import RedisClient
|
||||
from notifications_utils import logging, request_helper
|
||||
from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
|
||||
from werkzeug.local import LocalProxy
|
||||
|
||||
from app import config
|
||||
from app.clients import NotificationProviderClients
|
||||
from app.clients.cloudwatch.aws_cloudwatch import AwsCloudwatchClient
|
||||
from app.celery.celery import NotifyCelery
|
||||
from app.clients import Clients
|
||||
from app.clients.document_download import DocumentDownloadClient
|
||||
from app.clients.email.aws_ses import AwsSesClient
|
||||
from app.clients.email.aws_ses_stub import AwsSesStubClient
|
||||
from app.clients.pinpoint.aws_pinpoint import AwsPinpointClient
|
||||
from app.clients.sms.aws_sns import AwsSnsClient
|
||||
from notifications_utils import logging, request_helper
|
||||
from notifications_utils.clients.encryption.encryption_client import Encryption
|
||||
from notifications_utils.clients.redis.redis_client import RedisClient
|
||||
from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
|
||||
from app.clients.sms.firetext import FiretextClient
|
||||
from app.clients.sms.mmg import MMGClient
|
||||
from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
|
||||
from app.encryption import Encryption
|
||||
|
||||
job_cache = {}
|
||||
job_cache_lock = Lock()
|
||||
|
||||
|
||||
class NotifyCelery(Celery):
|
||||
def init_app(self, app):
|
||||
self.task_cls = make_task(app)
|
||||
|
||||
# Configure Celery app with options from the main app config.
|
||||
self.config_from_object(app.config["CELERY"])
|
||||
self.conf.worker_hijack_root_logger = False
|
||||
logger = real_logging.getLogger("celery")
|
||||
logger.propagate = False
|
||||
|
||||
def send_task(self, name, args=None, kwargs=None, **other_kwargs):
|
||||
other_kwargs["headers"] = other_kwargs.get("headers") or {}
|
||||
|
||||
if has_request_context() and hasattr(request, "request_id"):
|
||||
other_kwargs["headers"]["notify_request_id"] = request.request_id
|
||||
|
||||
elif has_app_context() and "request_id" in g:
|
||||
other_kwargs["headers"]["notify_request_id"] = g.request_id
|
||||
|
||||
return super().send_task(name, args, kwargs, **other_kwargs)
|
||||
DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
|
||||
DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
DATE_FORMAT = "%Y-%m-%d"
|
||||
|
||||
|
||||
class SQLAlchemy(_SQLAlchemy):
|
||||
"""We need to subclass SQLAlchemy in order to override create_engine options"""
|
||||
|
||||
def apply_driver_hacks(self, app, info, options):
|
||||
sa_url, options = super().apply_driver_hacks(app, info, options)
|
||||
|
||||
if "connect_args" not in options:
|
||||
options["connect_args"] = {}
|
||||
options["connect_args"]["options"] = "-c statement_timeout={}".format(
|
||||
int(app.config["SQLALCHEMY_STATEMENT_TIMEOUT"]) * 1000
|
||||
super().apply_driver_hacks(app, info, options)
|
||||
if 'connect_args' not in options:
|
||||
options['connect_args'] = {}
|
||||
options['connect_args']["options"] = "-c statement_timeout={}".format(
|
||||
int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
|
||||
)
|
||||
|
||||
return (sa_url, options)
|
||||
|
||||
|
||||
# Set db engine settings here for now.
|
||||
# They were not being set previous (despite environmental variables with appropriate
|
||||
# sounding names) and were defaulting to low values
|
||||
db = SQLAlchemy(
|
||||
engine_options={
|
||||
"pool_size": config.Config.SQLALCHEMY_POOL_SIZE,
|
||||
"max_overflow": 10,
|
||||
"pool_timeout": config.Config.SQLALCHEMY_POOL_TIMEOUT,
|
||||
"pool_recycle": config.Config.SQLALCHEMY_POOL_RECYCLE,
|
||||
"pool_pre_ping": True,
|
||||
}
|
||||
)
|
||||
db = SQLAlchemy()
|
||||
migrate = Migrate()
|
||||
ma = Marshmallow()
|
||||
notify_celery = NotifyCelery()
|
||||
firetext_client = FiretextClient()
|
||||
mmg_client = MMGClient()
|
||||
aws_ses_client = AwsSesClient()
|
||||
aws_ses_stub_client = AwsSesStubClient()
|
||||
aws_sns_client = AwsSnsClient()
|
||||
aws_cloudwatch_client = AwsCloudwatchClient()
|
||||
aws_pinpoint_client = AwsPinpointClient()
|
||||
encryption = Encryption()
|
||||
zendesk_client = ZendeskClient()
|
||||
statsd_client = StatsdClient()
|
||||
redis_store = RedisClient()
|
||||
performance_platform_client = PerformancePlatformClient()
|
||||
document_download_client = DocumentDownloadClient()
|
||||
|
||||
socketio = SocketIO(
|
||||
cors_allowed_origins=[
|
||||
config.Config.ADMIN_BASE_URL,
|
||||
],
|
||||
message_queue=config.Config.REDIS_URL,
|
||||
logger=True,
|
||||
engineio_logger=True,
|
||||
)
|
||||
clients = Clients()
|
||||
|
||||
notification_provider_clients = NotificationProviderClients()
|
||||
|
||||
api_user = LocalProxy(lambda: g.api_user)
|
||||
authenticated_service = LocalProxy(lambda: g.authenticated_service)
|
||||
api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
|
||||
authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
|
||||
|
||||
|
||||
def create_app(application):
|
||||
from app.config import configs
|
||||
|
||||
notify_environment = os.environ["NOTIFY_ENVIRONMENT"]
|
||||
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
|
||||
|
||||
application.config.from_object(configs[notify_environment])
|
||||
|
||||
application.config["NOTIFY_APP_NAME"] = application.name
|
||||
application.config['NOTIFY_APP_NAME'] = application.name
|
||||
init_app(application)
|
||||
|
||||
socketio.init_app(application)
|
||||
|
||||
from app.socket_handlers import register_socket_handlers
|
||||
|
||||
register_socket_handlers(socketio)
|
||||
request_helper.init_app(application)
|
||||
db.init_app(application)
|
||||
migrate.init_app(application, db=db)
|
||||
ma.init_app(application)
|
||||
zendesk_client.init_app(application)
|
||||
logging.init_app(application)
|
||||
aws_sns_client.init_app(application)
|
||||
|
||||
aws_ses_client.init_app()
|
||||
aws_ses_stub_client.init_app(stub_url=application.config["SES_STUB_URL"])
|
||||
aws_cloudwatch_client.init_app(application)
|
||||
aws_pinpoint_client.init_app(application)
|
||||
# If a stub url is provided for SES, then use the stub client rather than the real SES boto client
|
||||
email_clients = (
|
||||
[aws_ses_stub_client]
|
||||
if application.config["SES_STUB_URL"]
|
||||
else [aws_ses_client]
|
||||
)
|
||||
notification_provider_clients.init_app(
|
||||
sms_clients=[aws_sns_client], email_clients=email_clients
|
||||
)
|
||||
|
||||
statsd_client.init_app(application)
|
||||
logging.init_app(application, statsd_client)
|
||||
firetext_client.init_app(application, statsd_client=statsd_client)
|
||||
mmg_client.init_app(application, statsd_client=statsd_client)
|
||||
aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
|
||||
notify_celery.init_app(application)
|
||||
encryption.init_app(application)
|
||||
redis_store.init_app(application)
|
||||
performance_platform_client.init_app(application)
|
||||
document_download_client.init_app(application)
|
||||
clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client])
|
||||
|
||||
register_blueprint(application)
|
||||
register_v2_blueprints(application)
|
||||
|
||||
# avoid circular imports by importing this file later
|
||||
from app.commands import setup_commands
|
||||
|
||||
setup_commands(application)
|
||||
|
||||
# set up sqlalchemy events
|
||||
setup_sqlalchemy_events(application)
|
||||
|
||||
return application
|
||||
|
||||
|
||||
def register_blueprint(application):
|
||||
from app.authentication.auth import (
|
||||
requires_admin_auth,
|
||||
requires_auth,
|
||||
requires_no_auth,
|
||||
)
|
||||
from app.billing.rest import billing_blueprint
|
||||
from app.complaint.complaint_rest import complaint_blueprint
|
||||
from app.docs import docs as docs_blueprint
|
||||
from app.email_branding.rest import email_branding_blueprint
|
||||
from app.service.rest import service_blueprint
|
||||
from app.service.callback_rest import service_callback_blueprint
|
||||
from app.user.rest import user_blueprint
|
||||
from app.template.rest import template_blueprint
|
||||
from app.status.healthcheck import status as status_blueprint
|
||||
from app.job.rest import job_blueprint
|
||||
from app.notifications.rest import notifications as notifications_blueprint
|
||||
from app.invite.rest import invite as invite_blueprint
|
||||
from app.accept_invite.rest import accept_invite
|
||||
from app.template_statistics.rest import template_statistics as template_statistics_blueprint
|
||||
from app.events.rest import events as events_blueprint
|
||||
from app.provider_details.rest import provider_details as provider_details_blueprint
|
||||
from app.email_branding.rest import email_branding_blueprint
|
||||
from app.inbound_number.rest import inbound_number_blueprint
|
||||
from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
|
||||
from app.job.rest import job_blueprint
|
||||
from app.notifications.notifications_ses_callback import ses_callback_blueprint
|
||||
from app.notifications.receive_notifications import receive_notifications_blueprint
|
||||
from app.notifications.rest import notifications as notifications_blueprint
|
||||
from app.organization.invite_rest import organization_invite_blueprint
|
||||
from app.organization.rest import organization_blueprint
|
||||
from app.performance_dashboard.rest import performance_dashboard_blueprint
|
||||
from app.notifications.notifications_sms_callback import sms_callback_blueprint
|
||||
from app.notifications.notifications_letter_callback import letter_callback_blueprint
|
||||
from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
|
||||
from app.letters.rest import letter_job
|
||||
from app.billing.rest import billing_blueprint
|
||||
from app.organisation.rest import organisation_blueprint
|
||||
from app.organisation.invite_rest import organisation_invite_blueprint
|
||||
from app.complaint.complaint_rest import complaint_blueprint
|
||||
from app.platform_stats.rest import platform_stats_blueprint
|
||||
from app.provider_details.rest import provider_details as provider_details_blueprint
|
||||
from app.service.callback_rest import service_callback_blueprint
|
||||
from app.service.rest import service_blueprint
|
||||
from app.service_invite.rest import service_invite as service_invite_blueprint
|
||||
from app.status.healthcheck import status as status_blueprint
|
||||
from app.template.rest import template_blueprint
|
||||
from app.template_folder.rest import template_folder_blueprint
|
||||
from app.template_statistics.rest import (
|
||||
template_statistics as template_statistics_blueprint,
|
||||
)
|
||||
from app.letter_branding.letter_branding_rest import letter_branding_blueprint
|
||||
from app.upload.rest import upload_blueprint
|
||||
from app.user.rest import user_blueprint
|
||||
from app.webauthn.rest import webauthn_blueprint
|
||||
|
||||
service_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(service_blueprint, url_prefix="/service")
|
||||
application.register_blueprint(service_blueprint, url_prefix='/service')
|
||||
|
||||
user_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(user_blueprint, url_prefix="/user")
|
||||
|
||||
webauthn_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(webauthn_blueprint)
|
||||
application.register_blueprint(user_blueprint, url_prefix='/user')
|
||||
|
||||
template_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(template_blueprint)
|
||||
@@ -215,12 +139,10 @@ def register_blueprint(application):
|
||||
status_blueprint.before_request(requires_no_auth)
|
||||
application.register_blueprint(status_blueprint)
|
||||
|
||||
docs_blueprint.before_request(requires_no_auth)
|
||||
application.register_blueprint(docs_blueprint)
|
||||
|
||||
# delivery receipts
|
||||
ses_callback_blueprint.before_request(requires_no_auth)
|
||||
application.register_blueprint(ses_callback_blueprint)
|
||||
# TODO: make sure research mode can still trigger sms callbacks, then re-enable this
|
||||
sms_callback_blueprint.before_request(requires_no_auth)
|
||||
application.register_blueprint(sms_callback_blueprint)
|
||||
|
||||
# inbound sms
|
||||
receive_notifications_blueprint.before_request(requires_no_auth)
|
||||
@@ -232,11 +154,8 @@ def register_blueprint(application):
|
||||
job_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(job_blueprint)
|
||||
|
||||
service_invite_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(service_invite_blueprint)
|
||||
|
||||
organization_invite_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(organization_invite_blueprint)
|
||||
invite_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(invite_blueprint)
|
||||
|
||||
inbound_number_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(inbound_number_blueprint)
|
||||
@@ -244,6 +163,9 @@ def register_blueprint(application):
|
||||
inbound_sms_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(inbound_sms_blueprint)
|
||||
|
||||
accept_invite.before_request(requires_admin_auth)
|
||||
application.register_blueprint(accept_invite, url_prefix='/invite')
|
||||
|
||||
template_statistics_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(template_statistics_blueprint)
|
||||
|
||||
@@ -251,14 +173,16 @@ def register_blueprint(application):
|
||||
application.register_blueprint(events_blueprint)
|
||||
|
||||
provider_details_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(
|
||||
provider_details_blueprint, url_prefix="/provider-details"
|
||||
)
|
||||
application.register_blueprint(provider_details_blueprint, url_prefix='/provider-details')
|
||||
|
||||
email_branding_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(
|
||||
email_branding_blueprint, url_prefix="/email-branding"
|
||||
)
|
||||
application.register_blueprint(email_branding_blueprint, url_prefix='/email-branding')
|
||||
|
||||
letter_job.before_request(requires_admin_auth)
|
||||
application.register_blueprint(letter_job)
|
||||
|
||||
letter_callback_blueprint.before_request(requires_no_auth)
|
||||
application.register_blueprint(letter_callback_blueprint)
|
||||
|
||||
billing_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(billing_blueprint)
|
||||
@@ -266,28 +190,60 @@ def register_blueprint(application):
|
||||
service_callback_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(service_callback_blueprint)
|
||||
|
||||
organization_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(organization_blueprint, url_prefix="/organizations")
|
||||
organisation_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(organisation_blueprint, url_prefix='/organisations')
|
||||
|
||||
organisation_invite_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(organisation_invite_blueprint)
|
||||
|
||||
complaint_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(complaint_blueprint)
|
||||
|
||||
performance_dashboard_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(performance_dashboard_blueprint)
|
||||
|
||||
platform_stats_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(
|
||||
platform_stats_blueprint, url_prefix="/platform-stats"
|
||||
)
|
||||
application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')
|
||||
|
||||
template_folder_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(template_folder_blueprint)
|
||||
|
||||
letter_branding_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(letter_branding_blueprint)
|
||||
|
||||
upload_blueprint.before_request(requires_admin_auth)
|
||||
application.register_blueprint(upload_blueprint)
|
||||
|
||||
|
||||
def register_v2_blueprints(application):
|
||||
from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
|
||||
from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
|
||||
from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
|
||||
from app.v2.template.get_template import v2_template_blueprint as get_template
|
||||
from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
|
||||
from app.v2.template.post_template import v2_template_blueprint as post_template
|
||||
from app.authentication.auth import requires_auth
|
||||
|
||||
post_notifications.before_request(requires_auth)
|
||||
application.register_blueprint(post_notifications)
|
||||
|
||||
get_notifications.before_request(requires_auth)
|
||||
application.register_blueprint(get_notifications)
|
||||
|
||||
get_templates.before_request(requires_auth)
|
||||
application.register_blueprint(get_templates)
|
||||
|
||||
get_template.before_request(requires_auth)
|
||||
application.register_blueprint(get_template)
|
||||
|
||||
post_template.before_request(requires_auth)
|
||||
application.register_blueprint(post_template)
|
||||
|
||||
get_inbound_sms.before_request(requires_auth)
|
||||
application.register_blueprint(get_inbound_sms)
|
||||
|
||||
|
||||
def init_app(app):
|
||||
@app.before_request
|
||||
def record_user_agent():
|
||||
statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))
|
||||
|
||||
@app.before_request
|
||||
def record_request_details():
|
||||
@@ -296,44 +252,31 @@ def init_app(app):
|
||||
|
||||
@app.after_request
|
||||
def after_request(response):
|
||||
response.headers.add("X-Content-Type-Options", "nosniff")
|
||||
|
||||
# Some dynamic scan findings
|
||||
response.headers.add("Cross-Origin-Opener-Policy", "same-origin")
|
||||
response.headers.add("Cross-Origin-Embedder-Policy", "require-corp")
|
||||
response.headers.add("Cross-Origin-Resource-Policy", "same-origin")
|
||||
response.headers.add("Cross-Origin-Opener-Policy", "same-origin")
|
||||
|
||||
response.headers.add('Access-Control-Allow-Origin', '*')
|
||||
response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
|
||||
response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
|
||||
return response
|
||||
|
||||
@app.errorhandler(Exception)
|
||||
def exception(error):
|
||||
app.logger.exception(f"Handling error: {error}")
|
||||
app.logger.exception(error)
|
||||
# error.code is set for our exception types.
|
||||
msg = getattr(error, "message", str(error))
|
||||
code = getattr(error, "code", 500)
|
||||
response = make_response(
|
||||
jsonify(result="error", message=msg), code, error.get_headers()
|
||||
)
|
||||
response.content_type = "application/json"
|
||||
return response
|
||||
msg = getattr(error, 'message', str(error))
|
||||
code = getattr(error, 'code', 500)
|
||||
return jsonify(result='error', message=msg), code
|
||||
|
||||
@app.errorhandler(WerkzeugHTTPException)
|
||||
def werkzeug_exception(e):
|
||||
response = make_response(
|
||||
jsonify(result="error", message=e.description), e.code, e.get_headers()
|
||||
return make_response(
|
||||
jsonify(result='error', message=e.description),
|
||||
e.code,
|
||||
e.get_headers()
|
||||
)
|
||||
response.content_type = "application/json"
|
||||
return response
|
||||
|
||||
@app.errorhandler(404)
|
||||
def page_not_found(e):
|
||||
msg = e.description or "Not found"
|
||||
response = make_response(
|
||||
jsonify(result="error", message=msg), 404, e.get_headers()
|
||||
)
|
||||
response.content_type = "application/json"
|
||||
return response
|
||||
return jsonify(result='error', message=msg), 404
|
||||
|
||||
|
||||
def create_uuid():
|
||||
@@ -341,120 +284,16 @@ def create_uuid():
|
||||
|
||||
|
||||
def create_random_identifier():
|
||||
return "".join(
|
||||
secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16)
|
||||
)
|
||||
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
|
||||
|
||||
|
||||
# TODO maintainability what is the purpose of this? Debugging?
|
||||
def setup_sqlalchemy_events(app):
|
||||
# need this or db.engine isn't accessible
|
||||
with app.app_context():
|
||||
|
||||
@event.listens_for(db.engine, "connect")
|
||||
def connect(dbapi_connection, connection_record): # noqa
|
||||
pass
|
||||
|
||||
@event.listens_for(db.engine, "close")
|
||||
def close(dbapi_connection, connection_record): # noqa
|
||||
pass
|
||||
|
||||
@event.listens_for(db.engine, "checkout")
|
||||
def checkout(dbapi_connection, connection_record, connection_proxy): # noqa
|
||||
try:
|
||||
# this will overwrite any previous checkout_at timestamp
|
||||
connection_record.info["checkout_at"] = time.monotonic()
|
||||
|
||||
# checkin runs after the request is already torn down, therefore we add the request_data onto the
|
||||
# connection_record as otherwise it won't have that information when checkin actually runs.
|
||||
# Note: this is not a problem for checkouts as the checkout always happens within a web request or task
|
||||
|
||||
# web requests
|
||||
if has_request_context():
|
||||
connection_record.info["request_data"] = {
|
||||
"method": request.method,
|
||||
"host": request.host,
|
||||
"url_rule": (
|
||||
request.url_rule.rule if request.url_rule else "No endpoint"
|
||||
),
|
||||
}
|
||||
# celery apps
|
||||
elif current_task:
|
||||
connection_record.info["request_data"] = {
|
||||
"method": "celery",
|
||||
"host": current_app.config["NOTIFY_APP_NAME"], # worker name
|
||||
"url_rule": current_task.name, # task name
|
||||
}
|
||||
# anything else. migrations possibly, or flask cli commands.
|
||||
else:
|
||||
current_app.logger.warning(
|
||||
"Checked out sqlalchemy connection from outside of request/task"
|
||||
)
|
||||
connection_record.info["request_data"] = {
|
||||
"method": "unknown",
|
||||
"host": "unknown",
|
||||
"url_rule": "unknown",
|
||||
}
|
||||
except Exception:
|
||||
current_app.logger.exception(
|
||||
"Exception caught for checkout event.",
|
||||
)
|
||||
|
||||
@event.listens_for(db.engine, "checkin")
|
||||
def checkin(dbapi_connection, connection_record): # noqa
|
||||
pass
|
||||
|
||||
|
||||
def make_task(app):
|
||||
class NotifyTask(Task):
|
||||
abstract = True
|
||||
start = None
|
||||
|
||||
@property
|
||||
def queue_name(self):
|
||||
delivery_info = self.request.delivery_info or {}
|
||||
return delivery_info.get("routing_key", "none")
|
||||
|
||||
@property
|
||||
def request_id(self):
|
||||
# Note that each header is a direct attribute of the
|
||||
# task context (aka "request").
|
||||
return self.request.get("notify_request_id")
|
||||
|
||||
@contextmanager
|
||||
def app_context(self):
|
||||
with app.app_context():
|
||||
# Add 'request_id' to 'g' so that it gets logged.
|
||||
g.request_id = self.request_id
|
||||
yield
|
||||
|
||||
def on_success(self, retval, task_id, args, kwargs): # noqa
|
||||
# enables request id tracing for these logs
|
||||
with self.app_context():
|
||||
elapsed_time = time.monotonic() - self.start
|
||||
|
||||
app.logger.info(
|
||||
"Celery task {task_name} (queue: {queue_name}) took {time}".format(
|
||||
task_name=self.name,
|
||||
queue_name=self.queue_name,
|
||||
time="{0:.4f}".format(elapsed_time),
|
||||
)
|
||||
)
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, einfo): # noqa
|
||||
# enables request id tracing for these logs
|
||||
with self.app_context():
|
||||
app.logger.exception(
|
||||
"Celery task {task_name} (queue: {queue_name}) failed".format(
|
||||
task_name=self.name,
|
||||
queue_name=self.queue_name,
|
||||
),
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
# ensure task has flask context to access config, logger, etc
|
||||
with self.app_context():
|
||||
self.start = time.monotonic()
|
||||
return super().__call__(*args, **kwargs)
|
||||
|
||||
return NotifyTask
|
||||
def process_user_agent(user_agent_string):
|
||||
if user_agent_string and user_agent_string.lower().startswith("notify"):
|
||||
components = user_agent_string.split("/")
|
||||
client_name = components[0].lower()
|
||||
client_version = components[1].replace(".", "-")
|
||||
return "{}.{}".format(client_name, client_version)
|
||||
elif user_agent_string and not user_agent_string.lower().startswith("notify"):
|
||||
return "non-notify-user-agent"
|
||||
else:
|
||||
return "unknown"
|
||||
|
||||
52
app/accept_invite/rest.py
Normal file
52
app/accept_invite/rest.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from flask import (
|
||||
Blueprint,
|
||||
jsonify,
|
||||
current_app
|
||||
)
|
||||
|
||||
from itsdangerous import SignatureExpired, BadData
|
||||
|
||||
from notifications_utils.url_safe_token import check_token
|
||||
|
||||
from app.dao.invited_user_dao import get_invited_user_by_id
|
||||
from app.dao.organisation_dao import dao_get_invited_organisation_user
|
||||
|
||||
from app.errors import (
|
||||
register_errors,
|
||||
InvalidRequest
|
||||
)
|
||||
|
||||
from app.schemas import invited_user_schema
|
||||
|
||||
|
||||
accept_invite = Blueprint('accept_invite', __name__)
|
||||
register_errors(accept_invite)
|
||||
|
||||
|
||||
@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
|
||||
def validate_invitation_token(invitation_type, token):
|
||||
|
||||
max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
|
||||
|
||||
try:
|
||||
invited_user_id = check_token(token,
|
||||
current_app.config['SECRET_KEY'],
|
||||
current_app.config['DANGEROUS_SALT'],
|
||||
max_age_seconds)
|
||||
except SignatureExpired:
|
||||
errors = {'invitation':
|
||||
['Your invitation to GOV.UK Notify has expired. '
|
||||
'Please ask the person that invited you to send you another one']}
|
||||
raise InvalidRequest(errors, status_code=400)
|
||||
except BadData:
|
||||
errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
|
||||
raise InvalidRequest(errors, status_code=400)
|
||||
|
||||
if invitation_type == 'service':
|
||||
invited_user = get_invited_user_by_id(invited_user_id)
|
||||
return jsonify(data=invited_user_schema.dump(invited_user).data), 200
|
||||
elif invitation_type == 'organisation':
|
||||
invited_user = dao_get_invited_organisation_user(invited_user_id)
|
||||
return jsonify(data=invited_user.serialize()), 200
|
||||
else:
|
||||
raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))
|
||||
@@ -1,30 +1,13 @@
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from flask import current_app, g, request
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from app.serialised_models import SerialisedService
|
||||
from notifications_python_client.authentication import (
|
||||
decode_jwt_token,
|
||||
get_token_issuer,
|
||||
)
|
||||
from flask import request, _request_ctx_stack, current_app, g
|
||||
from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
|
||||
from notifications_python_client.errors import (
|
||||
TokenAlgorithmError,
|
||||
TokenDecodeError,
|
||||
TokenError,
|
||||
TokenExpiredError,
|
||||
TokenIssuerError,
|
||||
TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError
|
||||
)
|
||||
from notifications_utils import request_helper
|
||||
from sqlalchemy.exc import DataError
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
# stvnrlly - this is silly, but bandit has a multiline string bug (https://github.com/PyCQA/bandit/issues/658)
|
||||
# and flake8 wants a multiline quote here. TODO: check on bug status and restore sanity once possible
|
||||
TOKEN_MESSAGE_ONE = (
|
||||
"Invalid token: make sure your API token matches the example " # nosec B105
|
||||
)
|
||||
TOKEN_MESSAGE_TWO = "at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header" # nosec B105
|
||||
GENERAL_TOKEN_ERROR_MESSAGE = TOKEN_MESSAGE_ONE + TOKEN_MESSAGE_TWO
|
||||
from app.dao.services_dao import dao_fetch_service_by_id_with_api_keys
|
||||
|
||||
|
||||
class AuthError(Exception):
|
||||
@@ -36,22 +19,31 @@ class AuthError(Exception):
|
||||
self.api_key_id = api_key_id
|
||||
|
||||
def __str__(self):
|
||||
return "AuthError({message}, {code}, service_id={service_id}, api_key_id={api_key_id})".format(
|
||||
**self.__dict__
|
||||
)
|
||||
return 'AuthError({message}, {code}, service_id={service_id}, api_key_id={api_key_id})'.format(**self.__dict__)
|
||||
|
||||
def to_dict_v2(self):
|
||||
return {
|
||||
"status_code": self.code,
|
||||
"errors": [{"error": "AuthError", "message": self.short_message}],
|
||||
'status_code': self.code,
|
||||
"errors": [
|
||||
{
|
||||
"error": "AuthError",
|
||||
"message": self.short_message
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class InternalApiKey:
|
||||
def __init__(self, client_id, secret):
|
||||
self.secret = secret
|
||||
self.id = client_id
|
||||
self.expiry_date = None
|
||||
def get_auth_token(req):
|
||||
auth_header = req.headers.get('Authorization', None)
|
||||
if not auth_header:
|
||||
raise AuthError('Unauthorized, authentication token must be provided', 401)
|
||||
|
||||
auth_scheme = auth_header[:7].title()
|
||||
|
||||
if auth_scheme != 'Bearer ':
|
||||
raise AuthError('Unauthorized, authentication bearer scheme must be used', 401)
|
||||
|
||||
return auth_header[7:]
|
||||
|
||||
|
||||
def requires_no_auth():
|
||||
@@ -59,170 +51,80 @@ def requires_no_auth():
|
||||
|
||||
|
||||
def requires_admin_auth():
|
||||
requires_internal_auth(current_app.config.get("ADMIN_CLIENT_ID"))
|
||||
|
||||
|
||||
def requires_internal_auth(expected_client_id):
|
||||
|
||||
# Looks like we are hitting this for some reason
|
||||
# expected_client_id looks like ADMIN_CLIENT_USERNAME on the admin side, and
|
||||
# INTERNAL_CLIENT_API_KEYS is a dict
|
||||
keys = current_app.config.get("INTERNAL_CLIENT_API_KEYS")
|
||||
if keys.get(expected_client_id) is None:
|
||||
err_msg = "Unknown client_id for internal auth"
|
||||
current_app.logger.error(err_msg)
|
||||
raise TypeError(err_msg)
|
||||
|
||||
request_helper.check_proxy_header_before_request()
|
||||
auth_token = _get_auth_token(request)
|
||||
client_id = _get_token_issuer(auth_token)
|
||||
if client_id != expected_client_id:
|
||||
current_app.logger.info("client_id: %s", client_id)
|
||||
current_app.logger.info("expected_client_id: %s", expected_client_id)
|
||||
err_msg = "Unauthorized: not allowed to perform this action"
|
||||
current_app.logger.error(err_msg)
|
||||
raise AuthError(err_msg, 401)
|
||||
|
||||
api_keys = [
|
||||
InternalApiKey(client_id, secret)
|
||||
for secret in current_app.config.get("INTERNAL_CLIENT_API_KEYS")[client_id]
|
||||
]
|
||||
auth_token = get_auth_token(request)
|
||||
client = __get_token_issuer(auth_token)
|
||||
|
||||
_decode_jwt_token(auth_token, api_keys, client_id)
|
||||
g.service_id = client_id
|
||||
if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
|
||||
g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
|
||||
return handle_admin_key(auth_token, current_app.config.get('ADMIN_CLIENT_SECRET'))
|
||||
else:
|
||||
raise AuthError('Unauthorized, admin authentication token required', 401)
|
||||
|
||||
|
||||
def requires_auth():
|
||||
request_helper.check_proxy_header_before_request()
|
||||
|
||||
auth_token = _get_auth_token(request)
|
||||
issuer = _get_token_issuer(
|
||||
auth_token
|
||||
) # ie the `iss` claim which should be a service ID
|
||||
auth_token = get_auth_token(request)
|
||||
client = __get_token_issuer(auth_token)
|
||||
|
||||
try:
|
||||
service_id = uuid.UUID(issuer)
|
||||
except Exception:
|
||||
service = dao_fetch_service_by_id_with_api_keys(client)
|
||||
except DataError:
|
||||
raise AuthError("Invalid token: service id is not the right data type", 403)
|
||||
|
||||
try:
|
||||
service = SerialisedService.from_id(service_id)
|
||||
except NoResultFound:
|
||||
raise AuthError("Invalid token: service not found", 403)
|
||||
|
||||
if not service.api_keys:
|
||||
raise AuthError(
|
||||
"Invalid token: service has no API keys", 403, service_id=service.id
|
||||
)
|
||||
raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
|
||||
|
||||
if not service.active:
|
||||
raise AuthError(
|
||||
"Invalid token: service is archived", 403, service_id=service.id
|
||||
)
|
||||
raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
|
||||
|
||||
api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
|
||||
|
||||
current_app.logger.info(
|
||||
"API authorised for service {} with api key {}, using issuer {} for URL: {}".format(
|
||||
service_id, api_key.id, request.headers.get("User-Agent"), request.base_url
|
||||
)
|
||||
)
|
||||
|
||||
g.api_user = api_key
|
||||
g.service_id = service_id
|
||||
g.authenticated_service = service
|
||||
|
||||
|
||||
def _decode_jwt_token(auth_token, api_keys, service_id=None):
|
||||
# Temporary expedient to get e2e tests working. If we are in
|
||||
# the development or staging environments, just return the first
|
||||
# api key.
|
||||
if os.getenv("NOTIFY_ENVIRONMENT") in ["development", "staging"]:
|
||||
for api_key in api_keys:
|
||||
return api_key
|
||||
|
||||
for api_key in api_keys:
|
||||
for api_key in service.api_keys:
|
||||
try:
|
||||
decode_jwt_token(auth_token, api_key.secret)
|
||||
except TypeError:
|
||||
err_msg = "Invalid token: type error"
|
||||
current_app.logger.exception(err_msg)
|
||||
raise AuthError(
|
||||
"Invalid token: type error",
|
||||
403,
|
||||
service_id=service_id,
|
||||
api_key_id=api_key.id,
|
||||
)
|
||||
except TokenExpiredError:
|
||||
if not current_app.config.get("ALLOW_EXPIRED_API_TOKEN", False):
|
||||
err_msg = (
|
||||
"Error: Your system clock must be accurate to within 30 seconds"
|
||||
)
|
||||
current_app.logger.exception(err_msg)
|
||||
raise AuthError(
|
||||
err_msg, 403, service_id=service_id, api_key_id=api_key.id
|
||||
)
|
||||
err_msg = "Error: Your system clock must be accurate to within 30 seconds"
|
||||
raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
|
||||
except TokenAlgorithmError:
|
||||
err_msg = "Invalid token: algorithm used is not HS256"
|
||||
current_app.logger.exception(err_msg)
|
||||
raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
|
||||
raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
|
||||
except TokenDecodeError:
|
||||
# we attempted to validate the token but it failed meaning it was not signed using this api key.
|
||||
# Let's try the next one
|
||||
# TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
|
||||
# are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
|
||||
# continue on to check the next API key
|
||||
current_app.logger.exception(
|
||||
"TokenDecodeError. Couldn't decode auth token for given api key"
|
||||
)
|
||||
continue
|
||||
except TokenError:
|
||||
current_app.logger.exception("TokenError")
|
||||
# General error when trying to decode and validate the token
|
||||
raise AuthError(
|
||||
GENERAL_TOKEN_ERROR_MESSAGE,
|
||||
403,
|
||||
service_id=service_id,
|
||||
api_key_id=api_key.id,
|
||||
)
|
||||
|
||||
if api_key.expiry_date:
|
||||
err_msg = "Invalid token: API key revoked"
|
||||
current_app.logger.error(err_msg, exc_info=True)
|
||||
raise AuthError(
|
||||
err_msg,
|
||||
403,
|
||||
service_id=service_id,
|
||||
api_key_id=api_key.id,
|
||||
)
|
||||
raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
|
||||
|
||||
return api_key
|
||||
g.service_id = api_key.service_id
|
||||
_request_ctx_stack.top.authenticated_service = service
|
||||
_request_ctx_stack.top.api_user = api_key
|
||||
current_app.logger.info('API authorised for service {} with api key {}, using client {}'.format(
|
||||
service.id,
|
||||
api_key.id,
|
||||
request.headers.get('User-Agent')
|
||||
))
|
||||
return
|
||||
else:
|
||||
# service has API keys, but none matching the one the user provided
|
||||
# if we get here, we probably hit TokenDecodeErrors earlier
|
||||
err_msg = "Invalid token: API key not found"
|
||||
current_app.logger.error(err_msg, exc_info=True)
|
||||
raise AuthError(err_msg, 403, service_id=service_id)
|
||||
raise AuthError("Invalid token: signature, api token not found", 403, service_id=service.id)
|
||||
|
||||
|
||||
def _get_auth_token(req):
|
||||
auth_header = req.headers.get("Authorization", None)
|
||||
if not auth_header:
|
||||
raise AuthError("Unauthorized: authentication token must be provided", 401)
|
||||
|
||||
auth_scheme = auth_header[:7].title()
|
||||
|
||||
if auth_scheme != "Bearer ":
|
||||
raise AuthError("Unauthorized: authentication bearer scheme must be used", 401)
|
||||
|
||||
return auth_header[7:]
|
||||
|
||||
|
||||
def _get_token_issuer(auth_token):
|
||||
def __get_token_issuer(auth_token):
|
||||
try:
|
||||
issuer = get_token_issuer(auth_token)
|
||||
client = get_token_issuer(auth_token)
|
||||
except TokenIssuerError:
|
||||
raise AuthError("Invalid token: iss field not provided", 403)
|
||||
except TokenDecodeError:
|
||||
raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403)
|
||||
return issuer
|
||||
raise AuthError("Invalid token: signature, api token is not valid", 403)
|
||||
return client
|
||||
|
||||
|
||||
def handle_admin_key(auth_token, secret):
|
||||
try:
|
||||
decode_jwt_token(auth_token, secret)
|
||||
except TokenExpiredError:
|
||||
raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
|
||||
except TokenDecodeError:
|
||||
raise AuthError("Invalid token: signature, api token is not valid", 403)
|
||||
|
||||
618
app/aws/s3.py
618
app/aws/s3.py
@@ -1,577 +1,121 @@
|
||||
import csv
|
||||
import datetime
|
||||
import re
|
||||
import time
|
||||
from io import StringIO
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import botocore
|
||||
import eventlet
|
||||
from boto3 import Session
|
||||
from flask import current_app
|
||||
|
||||
from app import job_cache, job_cache_lock
|
||||
from app.clients import AWS_CLIENT_CONFIG
|
||||
import pytz
|
||||
from boto3 import client, resource
|
||||
import botocore
|
||||
|
||||
# from app.service.rest import get_service_by_id
|
||||
from app.utils import hilite
|
||||
from notifications_utils import aware_utcnow
|
||||
|
||||
FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
|
||||
NEW_FILE_LOCATION_STRUCTURE = "{}-service-notify/{}.csv"
|
||||
|
||||
# Temporarily extend cache to 7 days
|
||||
ttl = 60 * 60 * 24 * 7
|
||||
FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
|
||||
|
||||
|
||||
def get_service_id_from_key(key):
|
||||
key = key.replace("service-", "")
|
||||
key = key.split("/")
|
||||
key = key[0].replace("-notify", "")
|
||||
return key
|
||||
def get_s3_file(bucket_name, file_location):
|
||||
s3_file = get_s3_object(bucket_name, file_location)
|
||||
return s3_file.get()['Body'].read().decode('utf-8')
|
||||
|
||||
|
||||
def set_job_cache(key, value):
|
||||
# current_app.logger.debug(f"Setting {key} in the job_cache to {value}.")
|
||||
|
||||
with job_cache_lock:
|
||||
job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
|
||||
def get_s3_object(bucket_name, file_location):
|
||||
s3 = resource('s3')
|
||||
return s3.Object(bucket_name, file_location)
|
||||
|
||||
|
||||
def get_job_cache(key):
|
||||
|
||||
ret = job_cache.get(key)
|
||||
return ret
|
||||
|
||||
|
||||
def len_job_cache():
|
||||
ret = len(job_cache)
|
||||
current_app.logger.debug(f"Length of job_cache is {ret}")
|
||||
return ret
|
||||
|
||||
|
||||
def clean_cache():
|
||||
current_time = time.time()
|
||||
keys_to_delete = []
|
||||
for key, (_, expiry_time) in job_cache.items():
|
||||
if expiry_time < current_time:
|
||||
keys_to_delete.append(key)
|
||||
|
||||
current_app.logger.debug(
|
||||
f"Deleting the following keys from the job_cache: {keys_to_delete}"
|
||||
)
|
||||
with job_cache_lock:
|
||||
for key in keys_to_delete:
|
||||
del job_cache[key]
|
||||
|
||||
|
||||
def get_s3_client():
|
||||
|
||||
access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
|
||||
secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
|
||||
region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
|
||||
session = Session(
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
)
|
||||
current_app.logger.info(hilite("About to call session.client"))
|
||||
s3_client = session.client("s3", config=AWS_CLIENT_CONFIG)
|
||||
current_app.logger.info(hilite("SESSION CALLED"))
|
||||
return s3_client
|
||||
|
||||
|
||||
def get_s3_resource():
|
||||
access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
|
||||
secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
|
||||
region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
|
||||
session = Session(
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
)
|
||||
s3_resource = session.resource("s3", config=AWS_CLIENT_CONFIG)
|
||||
return s3_resource
|
||||
|
||||
|
||||
def _get_bucket_name():
|
||||
return current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
|
||||
|
||||
|
||||
def list_s3_objects():
|
||||
|
||||
bucket_name = _get_bucket_name()
|
||||
s3_client = get_s3_client()
|
||||
# Our reports only support 7 days, but pull 8 days to avoid
|
||||
# any edge cases
|
||||
time_limit = aware_utcnow() - datetime.timedelta(days=8)
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket_name)
|
||||
while True:
|
||||
for obj in response.get("Contents", []):
|
||||
if obj["LastModified"] >= time_limit:
|
||||
yield obj["Key"]
|
||||
if "NextContinuationToken" in response:
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=bucket_name,
|
||||
ContinuationToken=response["NextContinuationToken"],
|
||||
)
|
||||
else:
|
||||
break
|
||||
except Exception as e:
|
||||
current_app.logger.exception(
|
||||
f"An error occurred while regenerating cache #notify-debug-admin-1200: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
def get_bucket_name():
|
||||
return current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
|
||||
|
||||
|
||||
def cleanup_old_s3_objects():
|
||||
bucket_name = get_bucket_name()
|
||||
|
||||
s3_client = get_s3_client()
|
||||
# Our reports only support 7 days, but can be scheduled 3 days in advance
|
||||
# Use 14 day for the v1.0 version of this behavior
|
||||
time_limit = aware_utcnow() - datetime.timedelta(days=14)
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket_name)
|
||||
while True:
|
||||
for obj in response.get("Contents", []):
|
||||
if obj["LastModified"] <= time_limit:
|
||||
|
||||
try:
|
||||
remove_csv_object(obj["Key"])
|
||||
current_app.logger.debug(
|
||||
f"#delete-old-s3-objects Deleted: {obj['LastModified']} {obj['Key']}"
|
||||
)
|
||||
except botocore.exceptions.ClientError:
|
||||
current_app.logger.exception(f"Couldn't delete {obj['Key']}")
|
||||
|
||||
if "NextContinuationToken" in response:
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=bucket_name,
|
||||
ContinuationToken=response["NextContinuationToken"],
|
||||
)
|
||||
else:
|
||||
break
|
||||
except Exception:
|
||||
current_app.logger.exception(
|
||||
"#delete-old-s3-objects An error occurred while cleaning up old s3 objects",
|
||||
)
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket_name)
|
||||
|
||||
service_ids = set()
|
||||
while True:
|
||||
for obj in response.get("Contents", []):
|
||||
# Get the service id out of the upload key
|
||||
key = obj["Key"]
|
||||
object_arr = key.split("/")
|
||||
service_id = object_arr[0]
|
||||
service_id = service_id.replace("-service-notify", "")
|
||||
service_ids.add(service_id)
|
||||
if "NextContinuationToken" in response:
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=bucket_name,
|
||||
ContinuationToken=response["NextContinuationToken"],
|
||||
)
|
||||
else:
|
||||
break
|
||||
retained_services = []
|
||||
for service_id in service_ids:
|
||||
retained_services.append(service_id)
|
||||
|
||||
return service_ids
|
||||
except Exception as error:
|
||||
current_app.logger.exception(
|
||||
f"#delete-old-s3-objects An error occurred while cleaning up old s3 objects: {str(error)}"
|
||||
)
|
||||
|
||||
|
||||
def get_job_id_from_s3_object_key(key):
|
||||
object_arr = key.split("/")
|
||||
job_id = object_arr[1] # get the job_id
|
||||
job_id = job_id.replace(".csv", "") # we just want the job_id
|
||||
return job_id
|
||||
|
||||
|
||||
def read_s3_file(bucket_name, object_key, s3res):
|
||||
"""
|
||||
This method runs during the 'regenerate job cache' task.
|
||||
Note that in addition to retrieving the jobs and putting them
|
||||
into the cache, this method also does some pre-processing by
|
||||
putting a list of all phone numbers into the cache as well.
|
||||
|
||||
This means that when the report needs to be regenerated, it
|
||||
can easily find the phone numbers in the cache through job_cache[<job_id>_phones]
|
||||
and the personalization through job_cache[<job_id>_personalisation], which
|
||||
in theory should make report generation a lot faster.
|
||||
|
||||
We are moving processing from the front end where the user can see it
|
||||
in wait time, to this back end process.
|
||||
"""
|
||||
try:
|
||||
job_id = get_job_id_from_s3_object_key(object_key)
|
||||
service_id = get_service_id_from_key(object_key)
|
||||
|
||||
if get_job_cache(job_id) is None:
|
||||
job = (
|
||||
s3res.Object(bucket_name, object_key)
|
||||
.get()["Body"]
|
||||
.read()
|
||||
.decode("utf-8")
|
||||
)
|
||||
set_job_cache(job_id, job)
|
||||
set_job_cache(f"{job_id}_phones", extract_phones(job, service_id, job_id))
|
||||
set_job_cache(
|
||||
f"{job_id}_personalisation",
|
||||
extract_personalisation(job),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
current_app.logger.exception(str(e))
|
||||
|
||||
|
||||
def get_s3_files():
|
||||
"""
|
||||
We're using the ThreadPoolExecutor here to speed up the retrieval of S3
|
||||
csv files for scaling needs.
|
||||
"""
|
||||
bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
|
||||
object_keys = list_s3_objects()
|
||||
|
||||
s3res = get_s3_resource()
|
||||
current_app.logger.info(
|
||||
f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
|
||||
)
|
||||
count = 0
|
||||
try:
|
||||
for object_key in object_keys:
|
||||
read_s3_file(bucket_name, object_key, s3res)
|
||||
count = count + 1
|
||||
eventlet.sleep(0.2)
|
||||
except Exception:
|
||||
current_app.logger.exception(
|
||||
f"Trouble reading {object_key} which is # {count} during cache regeneration"
|
||||
)
|
||||
except OSError as e:
|
||||
current_app.logger.exception(
|
||||
f"Egress proxy issue reading {object_key} which is # {count}"
|
||||
)
|
||||
raise e
|
||||
|
||||
current_app.logger.info(
|
||||
f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200"
|
||||
)
|
||||
|
||||
|
||||
def get_s3_file(bucket_name, file_location, access_key, secret_key, region):
|
||||
s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
|
||||
return s3_file.get()["Body"].read().decode("utf-8")
|
||||
|
||||
|
||||
def download_from_s3(
|
||||
bucket_name, s3_key, local_filename, access_key, secret_key, region
|
||||
):
|
||||
|
||||
s3 = get_s3_client()
|
||||
result = None
|
||||
try:
|
||||
result = s3.download_file(bucket_name, s3_key, local_filename)
|
||||
current_app.logger.info(f"File downloaded successfully to {local_filename}")
|
||||
except botocore.exceptions.NoCredentialsError as nce:
|
||||
current_app.logger.exception("Credentials not found")
|
||||
raise Exception(nce)
|
||||
except botocore.exceptions.PartialCredentialsError as pce:
|
||||
current_app.logger.exception("Incomplete credentials provided")
|
||||
raise Exception(pce)
|
||||
except Exception:
|
||||
current_app.logger.exception("An error occurred")
|
||||
text = f"EXCEPTION local_filename {local_filename}"
|
||||
raise Exception(text)
|
||||
return result
|
||||
|
||||
|
||||
def get_s3_object(bucket_name, file_location, access_key, secret_key, region):
|
||||
|
||||
s3 = get_s3_resource()
|
||||
try:
|
||||
return s3.Object(bucket_name, file_location)
|
||||
except botocore.exceptions.ClientError:
|
||||
current_app.logger.exception(
|
||||
f"Can't retrieve S3 Object from {file_location}",
|
||||
)
|
||||
|
||||
|
||||
def purge_bucket(bucket_name, access_key, secret_key, region):
|
||||
s3 = get_s3_resource()
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
bucket.objects.all().delete()
|
||||
|
||||
|
||||
def file_exists(file_location):
|
||||
bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
|
||||
access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
|
||||
secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
|
||||
region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
|
||||
|
||||
def file_exists(bucket_name, file_location):
|
||||
try:
|
||||
# try and access metadata of object
|
||||
get_s3_object(
|
||||
bucket_name, file_location, access_key, secret_key, region
|
||||
).metadata
|
||||
get_s3_object(bucket_name, file_location).metadata
|
||||
return True
|
||||
except botocore.exceptions.ClientError as e:
|
||||
if e.response["ResponseMetadata"]["HTTPStatusCode"] == 404:
|
||||
if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
def get_job_location(service_id, job_id):
|
||||
|
||||
return (
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
|
||||
NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id),
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["region"],
|
||||
)
|
||||
|
||||
|
||||
def get_old_job_location(service_id, job_id):
|
||||
"""
|
||||
This is deprecated. We are transitioning to NEW_FILE_LOCATION_STRUCTURE,
|
||||
but it will take a few days where we have to support both formats.
|
||||
Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE.
|
||||
"""
|
||||
|
||||
return (
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
|
||||
current_app.config['CSV_UPLOAD_BUCKET_NAME'],
|
||||
FILE_LOCATION_STRUCTURE.format(service_id, job_id),
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["region"],
|
||||
)
|
||||
|
||||
|
||||
def get_job_and_metadata_from_s3(service_id, job_id):
|
||||
try:
|
||||
obj = get_s3_object(*get_job_location(service_id, job_id))
|
||||
except botocore.exceptions.ClientError:
|
||||
obj = get_s3_object(*get_old_job_location(service_id, job_id))
|
||||
|
||||
return obj.get()["Body"].read().decode("utf-8"), obj.get()["Metadata"]
|
||||
obj = get_s3_object(*get_job_location(service_id, job_id))
|
||||
return obj.get()['Body'].read().decode('utf-8'), obj.get()['Metadata']
|
||||
|
||||
|
||||
def get_job_from_s3(service_id, job_id):
|
||||
"""
|
||||
If and only if we hit a throttling exception of some kind, we want to try
|
||||
exponential backoff. However, if we are getting NoSuchKey or something
|
||||
that indicates things are permanently broken, we want to give up right away
|
||||
to save time.
|
||||
"""
|
||||
# We have to make sure the retries don't take up to much time, because
|
||||
# we might be retrieving dozens of jobs. So max time is:
|
||||
# 0.2 + 0.4 + 0.8 + 1.6 = 3.0 seconds
|
||||
retries = 0
|
||||
max_retries = 4
|
||||
backoff_factor = 0.2
|
||||
|
||||
if not file_exists(
|
||||
FILE_LOCATION_STRUCTURE.format(service_id, job_id)
|
||||
) and not file_exists(NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)):
|
||||
current_app.logger.error(
|
||||
f"This file with service_id {service_id} and job_id {job_id} does not exist"
|
||||
)
|
||||
return None
|
||||
|
||||
while retries < max_retries:
|
||||
|
||||
try:
|
||||
# TODO
|
||||
# for transition on optimizing the s3 partition, we have
|
||||
# to check for the file location using the new way and the
|
||||
# old way. After this has been on production for a few weeks
|
||||
# we should remove the check for the old way.
|
||||
try:
|
||||
obj = get_s3_object(*get_job_location(service_id, job_id))
|
||||
return obj.get()["Body"].read().decode("utf-8")
|
||||
except botocore.exceptions.ClientError:
|
||||
obj = get_s3_object(*get_old_job_location(service_id, job_id))
|
||||
return obj.get()["Body"].read().decode("utf-8")
|
||||
except botocore.exceptions.ClientError as e:
|
||||
if e.response["Error"]["Code"] in [
|
||||
"Throttling",
|
||||
"RequestTimeout",
|
||||
"SlowDown",
|
||||
]:
|
||||
current_app.logger.exception(
|
||||
f"Retrying job fetch service_id {service_id} job_id {job_id} retry_count={retries}",
|
||||
)
|
||||
retries += 1
|
||||
sleep_time = backoff_factor * (2**retries) # Exponential backoff
|
||||
eventlet.sleep(sleep_time)
|
||||
continue
|
||||
else:
|
||||
# Typically this is "NoSuchKey"
|
||||
current_app.logger.exception(
|
||||
f"Failed to get job with service_id {service_id} job_id {job_id}",
|
||||
)
|
||||
return None
|
||||
|
||||
except Exception:
|
||||
current_app.logger.exception(
|
||||
f"Failed to get job with service_id {service_id} job_id {job_id}retry_count={retries}",
|
||||
)
|
||||
return None
|
||||
|
||||
current_app.logger.error(
|
||||
f"Never retrieved job with service_id {service_id} job_id {job_id}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def extract_phones(job, service_id, job_id):
|
||||
job_csv_data = StringIO(job)
|
||||
csv_reader = csv.reader(job_csv_data)
|
||||
first_row = next(csv_reader)
|
||||
|
||||
phone_index = 0
|
||||
for i, item in enumerate(first_row):
|
||||
if item.lower().lstrip("\ufeff") == "phone number":
|
||||
phone_index = i
|
||||
break
|
||||
|
||||
phones = {}
|
||||
job_row = 0
|
||||
for row in csv_reader:
|
||||
|
||||
if phone_index >= len(row):
|
||||
phones[job_row] = "Unavailable"
|
||||
current_app.logger.error(
|
||||
f"Corrupt csv file, missing columns or\
|
||||
possibly a byte order mark in the file, \
|
||||
row: {row} service_id {service_id} job_id {job_id}",
|
||||
)
|
||||
# If the file is corrupt, stop trying to process it.
|
||||
return phones
|
||||
else:
|
||||
my_phone = row[phone_index]
|
||||
my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
|
||||
phones[job_row] = my_phone
|
||||
job_row = job_row + 1
|
||||
return phones
|
||||
|
||||
|
||||
def extract_personalisation(job):
|
||||
if isinstance(job, dict):
|
||||
job = job[0]
|
||||
job = job.split("\r\n")
|
||||
first_row = job[0]
|
||||
job.pop(0)
|
||||
first_row = first_row.split(",")
|
||||
personalisation = {}
|
||||
job_row = 0
|
||||
for row in job:
|
||||
row = row.split(",")
|
||||
temp = dict(zip(first_row, row))
|
||||
personalisation[job_row] = temp
|
||||
job_row = job_row + 1
|
||||
return personalisation
|
||||
|
||||
|
||||
def get_phone_number_from_s3(service_id, job_id, job_row_number):
|
||||
job = get_job_cache(job_id)
|
||||
if job is None:
|
||||
job = get_job_from_s3(service_id, job_id)
|
||||
# Even if it is None, put it here to avoid KeyErrors
|
||||
set_job_cache(job_id, job)
|
||||
else:
|
||||
# skip expiration date from cache, we don't need it here
|
||||
job = job[0]
|
||||
|
||||
if job is None:
|
||||
current_app.logger.error(
|
||||
f"Couldnt find phone for job with service_id {service_id} job_id {job_id} because job is missing"
|
||||
)
|
||||
return "Unavailable"
|
||||
|
||||
phones = get_job_cache(f"{job_id}_phones")
|
||||
if phones is None:
|
||||
current_app.logger.debug("HAVE TO REEXTRACT PHONES!")
|
||||
phones = extract_phones(job, service_id, job_id)
|
||||
set_job_cache(f"{job_id}_phones", phones)
|
||||
current_app.logger.debug(f"SETTING PHONES TO {phones}")
|
||||
else:
|
||||
phones = phones[
|
||||
0
|
||||
] # we only want the phone numbers not the cache expiration time
|
||||
|
||||
# If we can find the quick dictionary, use it
|
||||
phone_to_return = phones[job_row_number]
|
||||
if phone_to_return:
|
||||
return phone_to_return
|
||||
else:
|
||||
current_app.logger.warning(
|
||||
f"Was unable to retrieve phone number from lookup dictionary for job {job_id}"
|
||||
)
|
||||
return "Unavailable"
|
||||
|
||||
|
||||
def get_personalisation_from_s3(service_id, job_id, job_row_number):
|
||||
# We don't want to constantly pull down a job from s3 every time we need the personalisation.
|
||||
# At the same time we don't want to store it in redis or the db
|
||||
# So this is a little recycling mechanism to reduce the number of downloads.
|
||||
job = get_job_cache(job_id)
|
||||
if job is None:
|
||||
job = get_job_from_s3(service_id, job_id)
|
||||
# Even if it is None, put it here to avoid KeyErrors
|
||||
set_job_cache(job_id, job)
|
||||
else:
|
||||
# skip expiration date from cache, we don't need it here
|
||||
job = job[0]
|
||||
# If the job is None after our attempt to retrieve it from s3, it
|
||||
# probably means the job is old and has been deleted from s3, in
|
||||
# which case there is nothing we can do. It's unlikely to run into
|
||||
# this, but it could theoretically happen, especially if we ever
|
||||
# change the task schedules
|
||||
if job is None:
|
||||
current_app.logger.warning(
|
||||
f"Couldnt find personalisation for job_id {job_id} row number {job_row_number} because job is missing"
|
||||
)
|
||||
return {}
|
||||
|
||||
personalisation = get_job_cache(f"{job_id}_personalisation")
|
||||
if personalisation is None:
|
||||
set_job_cache(f"{job_id}_personalisation", extract_personalisation(job))
|
||||
|
||||
return get_job_cache(f"{job_id}_personalisation")[0].get(job_row_number)
|
||||
obj = get_s3_object(*get_job_location(service_id, job_id))
|
||||
return obj.get()['Body'].read().decode('utf-8')
|
||||
|
||||
|
||||
def get_job_metadata_from_s3(service_id, job_id):
|
||||
current_app.logger.debug(
|
||||
f"#notify-debug-s3-partitioning CALLING GET_JOB_METADATA with {service_id}, {job_id}"
|
||||
)
|
||||
obj = get_s3_object(*get_job_location(service_id, job_id))
|
||||
return obj.get()["Metadata"]
|
||||
return obj.get()['Metadata']
|
||||
|
||||
|
||||
def remove_job_from_s3(service_id, job_id):
|
||||
return remove_s3_object(*get_job_location(service_id, job_id))
|
||||
|
||||
|
||||
def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
|
||||
obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
|
||||
return obj.delete()
|
||||
|
||||
|
||||
def remove_csv_object(object_key):
|
||||
obj = get_s3_object(
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
|
||||
object_key,
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
|
||||
current_app.config["CSV_UPLOAD_BUCKET"]["region"],
|
||||
def get_s3_bucket_objects(bucket_name, subfolder='', older_than=7, limit_days=2):
|
||||
boto_client = client('s3', current_app.config['AWS_REGION'])
|
||||
paginator = boto_client.get_paginator('list_objects_v2')
|
||||
page_iterator = paginator.paginate(
|
||||
Bucket=bucket_name,
|
||||
Prefix=subfolder
|
||||
)
|
||||
|
||||
all_objects_in_bucket = []
|
||||
for page in page_iterator:
|
||||
if page.get('Contents'):
|
||||
all_objects_in_bucket.extend(page['Contents'])
|
||||
|
||||
return all_objects_in_bucket
|
||||
|
||||
|
||||
def filter_s3_bucket_objects_within_date_range(bucket_objects, older_than=7, limit_days=2):
|
||||
"""
|
||||
S3 returns the Object['LastModified'] as an 'offset-aware' timestamp so the
|
||||
date range filter must take this into account.
|
||||
|
||||
Additionally an additional Object is returned by S3 corresponding to the
|
||||
container directory. This is redundant and should be removed.
|
||||
|
||||
"""
|
||||
end_date = datetime.now(tz=pytz.utc) - timedelta(days=older_than)
|
||||
start_date = end_date - timedelta(days=limit_days)
|
||||
filtered_items = [item for item in bucket_objects if all([
|
||||
not item['Key'].endswith('/'),
|
||||
item['LastModified'] > start_date,
|
||||
item['LastModified'] < end_date
|
||||
])]
|
||||
|
||||
return filtered_items
|
||||
|
||||
|
||||
def remove_s3_object(bucket_name, object_key):
|
||||
obj = get_s3_object(bucket_name, object_key)
|
||||
return obj.delete()
|
||||
|
||||
|
||||
def remove_transformed_dvla_file(job_id):
|
||||
bucket_name = current_app.config['DVLA_BUCKETS']['job']
|
||||
file_location = '{}-dvla-job.text'.format(job_id)
|
||||
obj = get_s3_object(bucket_name, file_location)
|
||||
return obj.delete()
|
||||
|
||||
|
||||
def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None):
|
||||
s3_client = client('s3', current_app.config['AWS_REGION'])
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
page_iterator = paginator.paginate(
|
||||
Bucket=bucket_name,
|
||||
Prefix=subfolder
|
||||
)
|
||||
|
||||
for page in page_iterator:
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
if key.lower().endswith(suffix.lower()):
|
||||
if not last_modified or obj['LastModified'] >= last_modified:
|
||||
yield key
|
||||
|
||||
@@ -1,46 +1,42 @@
|
||||
from datetime import datetime
|
||||
|
||||
from app.enums import NotificationType
|
||||
|
||||
create_or_update_free_sms_fragment_limit_schema = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"description": "POST annual billing schema",
|
||||
"type": "object",
|
||||
"title": "Create",
|
||||
"properties": {
|
||||
"free_sms_fragment_limit": {"type": "integer", "minimum": 0},
|
||||
"free_sms_fragment_limit": {"type": "integer", "minimum": 1},
|
||||
},
|
||||
"required": ["free_sms_fragment_limit"],
|
||||
"required": ["free_sms_fragment_limit"]
|
||||
}
|
||||
|
||||
|
||||
def serialize_ft_billing_remove_emails(rows):
|
||||
return [
|
||||
{
|
||||
"month": (datetime.strftime(row.month, "%B")),
|
||||
"notification_type": row.notification_type,
|
||||
"chargeable_units": row.chargeable_units,
|
||||
"notifications_sent": row.notifications_sent,
|
||||
"rate": float(row.rate),
|
||||
"cost": float(row.cost),
|
||||
"free_allowance_used": row.free_allowance_used,
|
||||
"charged_units": row.charged_units,
|
||||
def serialize_ft_billing_remove_emails(data):
|
||||
results = []
|
||||
billed_notifications = [x for x in data if x.notification_type != 'email']
|
||||
for notification in billed_notifications:
|
||||
json_result = {
|
||||
"month": (datetime.strftime(notification.month, "%B")),
|
||||
"notification_type": notification.notification_type,
|
||||
"billing_units": notification.billable_units,
|
||||
"rate": float(notification.rate),
|
||||
"postage": notification.postage,
|
||||
}
|
||||
for row in rows
|
||||
if row.notification_type != NotificationType.EMAIL
|
||||
]
|
||||
results.append(json_result)
|
||||
return results
|
||||
|
||||
|
||||
def serialize_ft_billing_yearly_totals(rows):
|
||||
return [
|
||||
{
|
||||
"notification_type": row.notification_type,
|
||||
"chargeable_units": row.chargeable_units,
|
||||
"notifications_sent": row.notifications_sent,
|
||||
"rate": float(row.rate),
|
||||
"cost": float(row.cost),
|
||||
"free_allowance_used": row.free_allowance_used,
|
||||
"charged_units": row.charged_units,
|
||||
def serialize_ft_billing_yearly_totals(data):
|
||||
yearly_totals = []
|
||||
for total in data:
|
||||
json_result = {
|
||||
"notification_type": total.notification_type,
|
||||
"billing_units": total.billable_units,
|
||||
"rate": float(total.rate),
|
||||
"letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
yearly_totals.append(json_result)
|
||||
|
||||
return yearly_totals
|
||||
|
||||
@@ -1,105 +1,121 @@
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from app import db
|
||||
from app.billing.billing_schemas import (
|
||||
create_or_update_free_sms_fragment_limit_schema,
|
||||
serialize_ft_billing_remove_emails,
|
||||
serialize_ft_billing_yearly_totals,
|
||||
)
|
||||
from app.dao.annual_billing_dao import (
|
||||
dao_create_or_update_annual_billing_for_year,
|
||||
dao_get_free_sms_fragment_limit_for_year,
|
||||
dao_update_annual_billing_for_future_years,
|
||||
set_default_free_allowance_for_service,
|
||||
dao_get_all_free_sms_fragment_limit,
|
||||
dao_create_or_update_annual_billing_for_year,
|
||||
dao_update_annual_billing_for_future_years
|
||||
)
|
||||
from app.dao.date_util import get_current_calendar_year_start_year
|
||||
from app.dao.date_util import get_current_financial_year_start_year
|
||||
from app.dao.fact_billing_dao import (
|
||||
fetch_billing_totals_for_year,
|
||||
fetch_monthly_billing_for_year,
|
||||
fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
|
||||
)
|
||||
|
||||
from app.errors import InvalidRequest
|
||||
from app.errors import register_errors
|
||||
from app.models import Service
|
||||
from app.schema_validation import validate
|
||||
|
||||
billing_blueprint = Blueprint(
|
||||
"billing", __name__, url_prefix="/service/<uuid:service_id>/billing"
|
||||
'billing',
|
||||
__name__,
|
||||
url_prefix='/service/<uuid:service_id>/billing'
|
||||
)
|
||||
|
||||
|
||||
register_errors(billing_blueprint)
|
||||
|
||||
|
||||
@billing_blueprint.route("/monthly-usage")
|
||||
@billing_blueprint.route('/ft-monthly-usage')
|
||||
@billing_blueprint.route('/monthly-usage')
|
||||
def get_yearly_usage_by_monthly_from_ft_billing(service_id):
|
||||
try:
|
||||
year = int(request.args.get("year"))
|
||||
year = int(request.args.get('year'))
|
||||
except TypeError:
|
||||
return jsonify(result="error", message="No valid year provided"), 400
|
||||
return jsonify(result='error', message='No valid year provided'), 400
|
||||
results = fetch_monthly_billing_for_year(service_id=service_id, year=year)
|
||||
data = serialize_ft_billing_remove_emails(results)
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
@billing_blueprint.route("/yearly-usage-summary")
|
||||
@billing_blueprint.route('/ft-yearly-usage-summary')
|
||||
@billing_blueprint.route('/yearly-usage-summary')
|
||||
def get_yearly_billing_usage_summary_from_ft_billing(service_id):
|
||||
try:
|
||||
year = int(request.args.get("year"))
|
||||
year = int(request.args.get('year'))
|
||||
except TypeError:
|
||||
return jsonify(result="error", message="No valid year provided"), 400
|
||||
return jsonify(result='error', message='No valid year provided'), 400
|
||||
|
||||
billing_data = fetch_billing_totals_for_year(service_id, year)
|
||||
data = serialize_ft_billing_yearly_totals(billing_data)
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
@billing_blueprint.route("/free-sms-fragment-limit", methods=["GET"])
|
||||
@billing_blueprint.route('/free-sms-fragment-limit', methods=["GET"])
|
||||
def get_free_sms_fragment_limit(service_id):
|
||||
financial_year_start = request.args.get("financial_year_start")
|
||||
|
||||
annual_billing = dao_get_free_sms_fragment_limit_for_year(
|
||||
service_id, financial_year_start
|
||||
)
|
||||
financial_year_start = request.args.get('financial_year_start')
|
||||
|
||||
annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
|
||||
|
||||
if annual_billing is None:
|
||||
service = db.session.get(Service, service_id)
|
||||
# An entry does not exist in annual_billing table for that service and year.
|
||||
# Set the annual billing to the default free allowance based on the organization type of the service.
|
||||
# An entry does not exist in annual_billing table for that service and year. If it is a past year,
|
||||
# we return the oldest entry.
|
||||
# If it is the current or future years, we create an entry in the db table using the newest record,
|
||||
# and return that number. If all fails, we return InvalidRequest.
|
||||
sms_list = dao_get_all_free_sms_fragment_limit(service_id)
|
||||
|
||||
annual_billing = set_default_free_allowance_for_service(
|
||||
service=service,
|
||||
year_start=int(financial_year_start) if financial_year_start else None,
|
||||
)
|
||||
if not sms_list:
|
||||
raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
|
||||
else:
|
||||
if financial_year_start is None:
|
||||
financial_year_start = get_current_financial_year_start_year()
|
||||
|
||||
if int(financial_year_start) < get_current_financial_year_start_year():
|
||||
# return the earliest historical entry
|
||||
annual_billing = sms_list[0] # The oldest entry
|
||||
else:
|
||||
annual_billing = sms_list[-1] # The newest entry
|
||||
|
||||
annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
|
||||
annual_billing.free_sms_fragment_limit,
|
||||
financial_year_start)
|
||||
|
||||
return jsonify(annual_billing.serialize_free_sms_items()), 200
|
||||
|
||||
|
||||
@billing_blueprint.route("/free-sms-fragment-limit", methods=["POST"])
|
||||
@billing_blueprint.route('/free-sms-fragment-limit', methods=["POST"])
|
||||
def create_or_update_free_sms_fragment_limit(service_id):
|
||||
|
||||
req_args = request.get_json()
|
||||
|
||||
form = validate(req_args, create_or_update_free_sms_fragment_limit_schema)
|
||||
|
||||
update_free_sms_fragment_limit_data(
|
||||
service_id,
|
||||
free_sms_fragment_limit=form.get("free_sms_fragment_limit"),
|
||||
financial_year_start=form.get("financial_year_start"),
|
||||
)
|
||||
update_free_sms_fragment_limit_data(service_id,
|
||||
free_sms_fragment_limit=form.get('free_sms_fragment_limit'),
|
||||
financial_year_start=form.get('financial_year_start'))
|
||||
return jsonify(form), 201
|
||||
|
||||
|
||||
def update_free_sms_fragment_limit_data(
|
||||
service_id, free_sms_fragment_limit, financial_year_start
|
||||
):
|
||||
current_year = get_current_calendar_year_start_year()
|
||||
def update_free_sms_fragment_limit_data(service_id, free_sms_fragment_limit, financial_year_start):
|
||||
current_year = get_current_financial_year_start_year()
|
||||
if not financial_year_start:
|
||||
financial_year_start = current_year
|
||||
|
||||
dao_create_or_update_annual_billing_for_year(
|
||||
service_id, free_sms_fragment_limit, financial_year_start
|
||||
service_id,
|
||||
free_sms_fragment_limit,
|
||||
financial_year_start
|
||||
)
|
||||
# if we're trying to update historical data, don't touch other rows.
|
||||
# Otherwise, make sure that future years will get the new updated value.
|
||||
if financial_year_start >= current_year:
|
||||
dao_update_annual_billing_for_future_years(
|
||||
service_id, free_sms_fragment_limit, financial_year_start
|
||||
service_id,
|
||||
free_sms_fragment_limit,
|
||||
financial_year_start
|
||||
)
|
||||
|
||||
62
app/celery/celery.py
Normal file
62
app/celery/celery.py
Normal file
@@ -0,0 +1,62 @@
|
||||
import time
|
||||
|
||||
from celery import Celery, Task
|
||||
from celery.signals import worker_process_shutdown
|
||||
from flask import current_app, g, request
|
||||
from flask.ctx import has_request_context
|
||||
|
||||
|
||||
@worker_process_shutdown.connect
|
||||
def worker_process_shutdown(sender, signal, pid, exitcode, **kwargs):
|
||||
current_app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
|
||||
|
||||
|
||||
def make_task(app):
|
||||
class NotifyTask(Task):
|
||||
abstract = True
|
||||
start = None
|
||||
|
||||
def on_success(self, retval, task_id, args, kwargs):
|
||||
elapsed_time = time.time() - self.start
|
||||
app.logger.info(
|
||||
"{task_name} took {time}".format(
|
||||
task_name=self.name, time="{0:.4f}".format(elapsed_time)
|
||||
)
|
||||
)
|
||||
|
||||
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
||||
# ensure task will log exceptions to correct handlers
|
||||
app.logger.exception('Celery task: {} failed'.format(self.name))
|
||||
super().on_failure(exc, task_id, args, kwargs, einfo)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
# ensure task has flask context to access config, logger, etc
|
||||
with app.app_context():
|
||||
self.start = time.time()
|
||||
# Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
|
||||
# so that it gets logged
|
||||
g.request_id = kwargs.pop('request_id', None)
|
||||
return super().__call__(*args, **kwargs)
|
||||
|
||||
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
|
||||
link=None, link_error=None, **options):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
if has_request_context() and hasattr(request, 'request_id'):
|
||||
kwargs['request_id'] = request.request_id
|
||||
|
||||
return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
|
||||
|
||||
return NotifyTask
|
||||
|
||||
|
||||
class NotifyCelery(Celery):
|
||||
|
||||
def init_app(self, app):
|
||||
super().__init__(
|
||||
app.import_name,
|
||||
broker=app.config['BROKER_URL'],
|
||||
task_cls=make_task(app),
|
||||
)
|
||||
|
||||
self.conf.update(app.config)
|
||||
526
app/celery/letters_pdf_tasks.py
Normal file
526
app/celery/letters_pdf_tasks.py
Normal file
@@ -0,0 +1,526 @@
|
||||
import math
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
from hashlib import sha512
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
from botocore.exceptions import ClientError as BotoClientError
|
||||
from flask import current_app
|
||||
from requests import (
|
||||
post as requests_post,
|
||||
RequestException
|
||||
)
|
||||
from celery.exceptions import MaxRetriesExceededError
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from notifications_utils.s3 import s3upload
|
||||
|
||||
from app import notify_celery
|
||||
from app.aws import s3
|
||||
from app.config import QueueNames, TaskNames
|
||||
from app.dao.notifications_dao import (
|
||||
get_notification_by_id,
|
||||
update_notification_status_by_id,
|
||||
dao_update_notification,
|
||||
dao_get_notification_by_reference,
|
||||
dao_get_notifications_by_references,
|
||||
dao_update_notifications_by_reference,
|
||||
)
|
||||
from app.errors import VirusScanError
|
||||
from app.exceptions import NotificationTechnicalFailureException
|
||||
from app.letters.utils import (
|
||||
copy_redaction_failed_pdf,
|
||||
get_billable_units_for_letter_page_count,
|
||||
get_reference_from_filename,
|
||||
get_folder_name,
|
||||
upload_letter_pdf,
|
||||
ScanErrorType,
|
||||
move_failed_pdf,
|
||||
move_sanitised_letter_to_test_or_live_pdf_bucket,
|
||||
move_scan_to_invalid_pdf_bucket,
|
||||
move_error_pdf_to_scan_bucket,
|
||||
get_file_names_from_error_bucket,
|
||||
)
|
||||
from app.models import (
|
||||
KEY_TYPE_TEST,
|
||||
NOTIFICATION_CREATED,
|
||||
NOTIFICATION_DELIVERED,
|
||||
NOTIFICATION_PENDING_VIRUS_CHECK,
|
||||
NOTIFICATION_TECHNICAL_FAILURE,
|
||||
NOTIFICATION_VALIDATION_FAILED,
|
||||
NOTIFICATION_VIRUS_SCAN_FAILED,
|
||||
)
|
||||
from app.cronitor import cronitor
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def create_letters_pdf(self, notification_id):
|
||||
try:
|
||||
notification = get_notification_by_id(notification_id, _raise=True)
|
||||
pdf_data, billable_units = get_letters_pdf(
|
||||
notification.template,
|
||||
contact_block=notification.reply_to_text,
|
||||
filename=notification.service.letter_branding and notification.service.letter_branding.filename,
|
||||
values=notification.personalisation
|
||||
)
|
||||
|
||||
upload_letter_pdf(notification, pdf_data)
|
||||
|
||||
if notification.key_type != KEY_TYPE_TEST:
|
||||
notification.billable_units = billable_units
|
||||
dao_update_notification(notification)
|
||||
|
||||
current_app.logger.info(
|
||||
'Letter notification reference {reference}: billable units set to {billable_units}'.format(
|
||||
reference=str(notification.reference), billable_units=billable_units))
|
||||
|
||||
except (RequestException, BotoClientError):
|
||||
try:
|
||||
current_app.logger.exception(
|
||||
"Letters PDF notification creation for id: {} failed".format(notification_id)
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except MaxRetriesExceededError:
|
||||
current_app.logger.error(
|
||||
"RETRY FAILED: task create_letters_pdf failed for notification {}".format(notification_id),
|
||||
)
|
||||
update_notification_status_by_id(notification_id, 'technical-failure')
|
||||
|
||||
|
||||
def get_letters_pdf(template, contact_block, filename, values):
|
||||
template_for_letter_print = {
|
||||
"subject": template.subject,
|
||||
"content": template.content
|
||||
}
|
||||
|
||||
data = {
|
||||
'letter_contact_block': contact_block,
|
||||
'template': template_for_letter_print,
|
||||
'values': values,
|
||||
'filename': filename,
|
||||
}
|
||||
resp = requests_post(
|
||||
'{}/print.pdf'.format(
|
||||
current_app.config['TEMPLATE_PREVIEW_API_HOST']
|
||||
),
|
||||
json=data,
|
||||
headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
pages_per_sheet = 2
|
||||
billable_units = math.ceil(int(resp.headers.get("X-pdf-page-count", 0)) / pages_per_sheet)
|
||||
|
||||
return resp.content, billable_units
|
||||
|
||||
|
||||
@notify_celery.task(name='collate-letter-pdfs-for-day')
|
||||
@cronitor("collate-letter-pdfs-for-day")
|
||||
def collate_letter_pdfs_for_day(date=None):
|
||||
if not date:
|
||||
# Using the truncated date is ok because UTC to BST does not make a difference to the date,
|
||||
# since it is triggered mid afternoon.
|
||||
date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
|
||||
letter_pdfs = sorted(
|
||||
s3.get_s3_bucket_objects(
|
||||
current_app.config['LETTERS_PDF_BUCKET_NAME'],
|
||||
subfolder=date
|
||||
),
|
||||
key=lambda letter: letter['Key']
|
||||
)
|
||||
for i, letters in enumerate(group_letters(letter_pdfs)):
|
||||
filenames = [letter['Key'] for letter in letters]
|
||||
|
||||
hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
|
||||
# eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
|
||||
dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
|
||||
date=date,
|
||||
num=i + 1,
|
||||
hash=hash
|
||||
)
|
||||
|
||||
current_app.logger.info(
|
||||
'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
|
||||
len(filenames),
|
||||
dvla_filename,
|
||||
sum(letter['Size'] for letter in letters)
|
||||
)
|
||||
)
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
|
||||
kwargs={
|
||||
'filenames_to_zip': filenames,
|
||||
'upload_filename': dvla_filename
|
||||
},
|
||||
queue=QueueNames.PROCESS_FTP,
|
||||
compression='zlib'
|
||||
)
|
||||
|
||||
|
||||
def group_letters(letter_pdfs):
|
||||
"""
|
||||
Group letters in chunks of MAX_LETTER_PDF_ZIP_FILESIZE. Will add files to lists, never going over that size.
|
||||
If a single file is (somehow) larger than MAX_LETTER_PDF_ZIP_FILESIZE that'll be in a list on it's own.
|
||||
If there are no files, will just exit (rather than yielding an empty list).
|
||||
"""
|
||||
running_filesize = 0
|
||||
list_of_files = []
|
||||
for letter in letter_pdfs:
|
||||
if letter['Key'].lower().endswith('.pdf') and letter_in_created_state(letter['Key']):
|
||||
if (
|
||||
running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
|
||||
len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
|
||||
):
|
||||
yield list_of_files
|
||||
running_filesize = 0
|
||||
list_of_files = []
|
||||
|
||||
running_filesize += letter['Size']
|
||||
list_of_files.append(letter)
|
||||
|
||||
if list_of_files:
|
||||
yield list_of_files
|
||||
|
||||
|
||||
def letter_in_created_state(filename):
|
||||
# filename looks like '2018-01-13/NOTIFY.ABCDEF1234567890.D.2.C.C.20180113120000.PDF'
|
||||
subfolder = filename.split('/')[0]
|
||||
ref = get_reference_from_filename(filename)
|
||||
notifications = dao_get_notifications_by_references([ref])
|
||||
if notifications:
|
||||
if notifications[0].status == NOTIFICATION_CREATED:
|
||||
return True
|
||||
current_app.logger.info('Collating letters for {} but notification with reference {} already in {}'.format(
|
||||
subfolder,
|
||||
ref,
|
||||
notifications[0].status
|
||||
))
|
||||
return False
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='process-virus-scan-passed', max_retries=15, default_retry_delay=300)
|
||||
def process_virus_scan_passed(self, filename):
|
||||
reference = get_reference_from_filename(filename)
|
||||
notification = dao_get_notification_by_reference(reference)
|
||||
current_app.logger.info('notification id {} Virus scan passed: {}'.format(notification.id, filename))
|
||||
|
||||
is_test_key = notification.key_type == KEY_TYPE_TEST
|
||||
|
||||
scan_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
|
||||
old_pdf = scan_pdf_object.get()['Body'].read()
|
||||
|
||||
sanitise_response, result = _sanitise_precompiled_pdf(self, notification, old_pdf)
|
||||
new_pdf = None
|
||||
if result == 'validation_passed':
|
||||
new_pdf = base64.b64decode(sanitise_response["file"].encode())
|
||||
|
||||
redaction_failed_message = sanitise_response.get("redaction_failed_message")
|
||||
if redaction_failed_message and not is_test_key:
|
||||
current_app.logger.info('{} for notification id {} ({})'.format(
|
||||
redaction_failed_message, notification.id, filename)
|
||||
)
|
||||
copy_redaction_failed_pdf(filename)
|
||||
|
||||
billable_units = get_billable_units_for_letter_page_count(sanitise_response.get("page_count"))
|
||||
|
||||
# TODO: Remove this once CYSP update their template to not cross over the margins
|
||||
if notification.service_id == UUID('fe44178f-3b45-4625-9f85-2264a36dd9ec'): # CYSP
|
||||
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
|
||||
new_pdf = old_pdf
|
||||
|
||||
if result == 'validation_failed' and not new_pdf:
|
||||
current_app.logger.info('Invalid precompiled pdf received {} ({})'.format(notification.id, filename))
|
||||
_move_invalid_letter_and_update_status(
|
||||
notification=notification,
|
||||
filename=filename,
|
||||
scan_pdf_object=scan_pdf_object,
|
||||
message=sanitise_response["message"],
|
||||
invalid_pages=sanitise_response.get("invalid_pages"),
|
||||
page_count=sanitise_response.get("page_count")
|
||||
)
|
||||
return
|
||||
|
||||
current_app.logger.info('notification id {} ({}) sanitised and ready to send'.format(notification.id, filename))
|
||||
|
||||
try:
|
||||
_upload_pdf_to_test_or_live_pdf_bucket(
|
||||
new_pdf,
|
||||
filename,
|
||||
is_test_letter=is_test_key,
|
||||
created_at=notification.created_at
|
||||
)
|
||||
|
||||
update_letter_pdf_status(
|
||||
reference=reference,
|
||||
status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
|
||||
billable_units=billable_units,
|
||||
recipient_address=sanitise_response.get("recipient_address")
|
||||
)
|
||||
scan_pdf_object.delete()
|
||||
except BotoClientError:
|
||||
current_app.logger.exception(
|
||||
"Error uploading letter to live pdf bucket for notification: {}".format(notification.id)
|
||||
)
|
||||
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300)
|
||||
def sanitise_letter(self, filename):
|
||||
try:
|
||||
reference = get_reference_from_filename(filename)
|
||||
notification = dao_get_notification_by_reference(reference)
|
||||
|
||||
current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))
|
||||
|
||||
if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
|
||||
current_app.logger.info('Sanitise letter called for notification {} which has is in {} state'.format(
|
||||
notification.id, notification.status))
|
||||
return
|
||||
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.SANITISE_LETTER,
|
||||
kwargs={
|
||||
'notification_id': str(notification.id),
|
||||
'filename': filename,
|
||||
},
|
||||
queue=QueueNames.SANITISE_LETTERS,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
current_app.logger.exception(
|
||||
"RETRY: calling sanitise_letter task for notification {} failed".format(notification.id)
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
message = "RETRY FAILED: Max retries reached. " \
|
||||
"The task sanitise_letter failed for notification {}. " \
|
||||
"Notification has been updated to technical-failure".format(notification.id)
|
||||
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
raise NotificationTechnicalFailureException(message)
|
||||
|
||||
|
||||
@notify_celery.task(name='process-sanitised-letter')
|
||||
def process_sanitised_letter(
|
||||
page_count,
|
||||
message,
|
||||
invalid_pages,
|
||||
validation_status,
|
||||
filename,
|
||||
notification_id,
|
||||
):
|
||||
current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id))
|
||||
notification = get_notification_by_id(notification_id, _raise=True)
|
||||
|
||||
if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
|
||||
current_app.logger.info(
|
||||
'process-sanitised-letter task called for notification {} which has is in {} state'.format(
|
||||
notification.id, notification.status)
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
|
||||
|
||||
if validation_status == 'failed':
|
||||
current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format(
|
||||
notification_id, filename))
|
||||
|
||||
_move_invalid_letter_and_update_status(
|
||||
notification=notification,
|
||||
filename=filename,
|
||||
scan_pdf_object=original_pdf_object,
|
||||
message=message,
|
||||
invalid_pages=invalid_pages,
|
||||
page_count=page_count,
|
||||
)
|
||||
return
|
||||
|
||||
current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format(
|
||||
notification_id, filename))
|
||||
|
||||
billable_units = get_billable_units_for_letter_page_count(page_count)
|
||||
is_test_key = notification.key_type == KEY_TYPE_TEST
|
||||
|
||||
move_sanitised_letter_to_test_or_live_pdf_bucket(filename, is_test_key, notification.created_at)
|
||||
# We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
|
||||
original_pdf_object.delete()
|
||||
update_letter_pdf_status(
|
||||
reference=notification.reference,
|
||||
status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
|
||||
billable_units=billable_units
|
||||
)
|
||||
|
||||
except BotoClientError:
|
||||
current_app.logger.exception(
|
||||
"Boto error when processing sanitised letter for notification {}".format(filename, notification.id)
|
||||
)
|
||||
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
raise NotificationTechnicalFailureException
|
||||
|
||||
|
||||
def _move_invalid_letter_and_update_status(
|
||||
*, notification, filename, scan_pdf_object, message=None, invalid_pages=None, page_count=None
|
||||
):
|
||||
try:
|
||||
move_scan_to_invalid_pdf_bucket(
|
||||
source_filename=filename,
|
||||
message=message,
|
||||
invalid_pages=invalid_pages,
|
||||
page_count=page_count
|
||||
)
|
||||
scan_pdf_object.delete()
|
||||
|
||||
update_letter_pdf_status(
|
||||
reference=notification.reference,
|
||||
status=NOTIFICATION_VALIDATION_FAILED,
|
||||
billable_units=0)
|
||||
except BotoClientError:
|
||||
current_app.logger.exception(
|
||||
"Error when moving letter with id {} to invalid PDF bucket".format(notification.id)
|
||||
)
|
||||
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
raise NotificationTechnicalFailureException
|
||||
|
||||
|
||||
def _upload_pdf_to_test_or_live_pdf_bucket(pdf_data, filename, is_test_letter, created_at):
|
||||
target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME'
|
||||
target_bucket_name = current_app.config[target_bucket_config]
|
||||
target_filename = get_folder_name(created_at, dont_use_sending_date=is_test_letter) + filename
|
||||
|
||||
s3upload(
|
||||
filedata=pdf_data,
|
||||
region=current_app.config['AWS_REGION'],
|
||||
bucket_name=target_bucket_name,
|
||||
file_location=target_filename
|
||||
)
|
||||
|
||||
|
||||
def _sanitise_precompiled_pdf(self, notification, precompiled_pdf):
|
||||
try:
|
||||
response = requests_post(
|
||||
'{}/precompiled/sanitise'.format(
|
||||
current_app.config['TEMPLATE_PREVIEW_API_HOST']
|
||||
),
|
||||
data=precompiled_pdf,
|
||||
headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY']),
|
||||
'Service-ID': str(notification.service_id),
|
||||
'Notification-ID': str(notification.id)}
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json(), "validation_passed"
|
||||
except RequestException as ex:
|
||||
if ex.response is not None and ex.response.status_code == 400:
|
||||
message = "sanitise_precompiled_pdf validation error for notification: {}. ".format(notification.id)
|
||||
if response.json().get("message"):
|
||||
message += response.json()["message"]
|
||||
if response.json().get("invalid_pages"):
|
||||
message += (" on pages: " + ", ".join(map(str, response.json()["invalid_pages"])))
|
||||
|
||||
current_app.logger.info(
|
||||
message
|
||||
)
|
||||
return response.json(), "validation_failed"
|
||||
|
||||
try:
|
||||
current_app.logger.exception(
|
||||
"sanitise_precompiled_pdf failed for notification: {}".format(notification.id)
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except MaxRetriesExceededError:
|
||||
current_app.logger.error(
|
||||
"RETRY FAILED: sanitise_precompiled_pdf failed for notification {}".format(notification.id),
|
||||
)
|
||||
|
||||
notification.status = NOTIFICATION_TECHNICAL_FAILURE
|
||||
dao_update_notification(notification)
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name='process-virus-scan-failed')
|
||||
def process_virus_scan_failed(filename):
|
||||
move_failed_pdf(filename, ScanErrorType.FAILURE)
|
||||
reference = get_reference_from_filename(filename)
|
||||
notification = dao_get_notification_by_reference(reference)
|
||||
updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0)
|
||||
|
||||
if updated_count != 1:
|
||||
raise Exception(
|
||||
"There should only be one letter notification for each reference. Found {} notifications".format(
|
||||
updated_count
|
||||
)
|
||||
)
|
||||
|
||||
error = VirusScanError('notification id {} Virus scan failed: {}'.format(notification.id, filename))
|
||||
current_app.logger.exception(error)
|
||||
raise error
|
||||
|
||||
|
||||
@notify_celery.task(name='process-virus-scan-error')
|
||||
def process_virus_scan_error(filename):
|
||||
move_failed_pdf(filename, ScanErrorType.ERROR)
|
||||
reference = get_reference_from_filename(filename)
|
||||
notification = dao_get_notification_by_reference(reference)
|
||||
updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE, billable_units=0)
|
||||
|
||||
if updated_count != 1:
|
||||
raise Exception(
|
||||
"There should only be one letter notification for each reference. Found {} notifications".format(
|
||||
updated_count
|
||||
)
|
||||
)
|
||||
error = VirusScanError('notification id {} Virus scan error: {}'.format(notification.id, filename))
|
||||
current_app.logger.exception(error)
|
||||
raise error
|
||||
|
||||
|
||||
def update_letter_pdf_status(reference, status, billable_units, recipient_address=None):
|
||||
|
||||
update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()}
|
||||
if recipient_address:
|
||||
update_dict['to'] = recipient_address
|
||||
return dao_update_notifications_by_reference(
|
||||
references=[reference],
|
||||
update_dict=update_dict)[0]
|
||||
|
||||
|
||||
def replay_letters_in_error(filename=None):
|
||||
# This method can be used to replay letters that end up in the ERROR directory.
|
||||
# We had an incident where clamAV was not processing the virus scan.
|
||||
if filename:
|
||||
move_error_pdf_to_scan_bucket(filename)
|
||||
# call task to add the filename to anti virus queue
|
||||
current_app.logger.info("Calling scan_file for: {}".format(filename))
|
||||
|
||||
if current_app.config['ANTIVIRUS_ENABLED']:
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.SCAN_FILE,
|
||||
kwargs={'filename': filename},
|
||||
queue=QueueNames.ANTIVIRUS,
|
||||
)
|
||||
else:
|
||||
# stub out antivirus in dev
|
||||
process_virus_scan_passed.apply_async(
|
||||
kwargs={'filename': filename},
|
||||
queue=QueueNames.LETTERS,
|
||||
)
|
||||
else:
|
||||
error_files = get_file_names_from_error_bucket()
|
||||
for item in error_files:
|
||||
moved_file_name = item.key.split('/')[1]
|
||||
current_app.logger.info("Calling scan_file for: {}".format(moved_file_name))
|
||||
move_error_pdf_to_scan_bucket(moved_file_name)
|
||||
# call task to add the filename to anti virus queue
|
||||
if current_app.config['ANTIVIRUS_ENABLED']:
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.SCAN_FILE,
|
||||
kwargs={'filename': moved_file_name},
|
||||
queue=QueueNames.ANTIVIRUS,
|
||||
)
|
||||
else:
|
||||
# stub out antivirus in dev
|
||||
process_virus_scan_passed.apply_async(
|
||||
kwargs={'filename': moved_file_name},
|
||||
queue=QueueNames.LETTERS,
|
||||
)
|
||||
@@ -1,37 +1,57 @@
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import (
|
||||
datetime,
|
||||
timedelta
|
||||
)
|
||||
|
||||
import pytz
|
||||
from flask import current_app
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from app import notify_celery
|
||||
from app import notify_celery, performance_platform_client, zendesk_client
|
||||
from app.aws import s3
|
||||
from app.aws.s3 import remove_csv_object
|
||||
from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
|
||||
from app.celery.service_callback_tasks import (
|
||||
send_delivery_status_to_service,
|
||||
create_delivery_status_callback_data,
|
||||
)
|
||||
from app.config import QueueNames
|
||||
from app.dao.fact_processing_time_dao import insert_update_processing_time
|
||||
from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
|
||||
from app.dao.jobs_dao import (
|
||||
dao_archive_job,
|
||||
dao_get_jobs_older_than_data_retention,
|
||||
dao_get_unfinished_jobs,
|
||||
dao_archive_job
|
||||
)
|
||||
from app.dao.notifications_dao import (
|
||||
dao_get_notifications_processing_time_stats,
|
||||
dao_timeout_notifications,
|
||||
get_service_ids_with_notifications_before,
|
||||
move_notifications_to_notification_history,
|
||||
delete_notifications_older_than_retention_by_type,
|
||||
)
|
||||
from app.dao.service_data_retention_dao import (
|
||||
fetch_service_data_retention_for_all_services_by_notification_type,
|
||||
from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
|
||||
from app.exceptions import NotificationTechnicalFailureException
|
||||
from app.models import (
|
||||
Notification,
|
||||
NOTIFICATION_SENDING,
|
||||
EMAIL_TYPE,
|
||||
SMS_TYPE,
|
||||
LETTER_TYPE,
|
||||
KEY_TYPE_NORMAL
|
||||
)
|
||||
from app.enums import NotificationType
|
||||
from app.models import FactProcessingTime
|
||||
from app.utils import get_midnight_in_utc, utc_now
|
||||
from app.performance_platform import total_sent_notifications, processing_time
|
||||
from app.cronitor import cronitor
|
||||
from app.utils import get_london_midnight_in_utc
|
||||
|
||||
|
||||
@notify_celery.task(name="remove-sms-email-jobs")
|
||||
@notify_celery.task(name="remove_sms_email_jobs")
|
||||
@cronitor("remove_sms_email_jobs")
|
||||
@statsd(namespace="tasks")
|
||||
def remove_sms_email_csv_files():
|
||||
_remove_csv_files([NotificationType.EMAIL, NotificationType.SMS])
|
||||
_remove_csv_files([EMAIL_TYPE, SMS_TYPE])
|
||||
|
||||
|
||||
@notify_celery.task(name="remove_letter_jobs")
|
||||
@cronitor("remove_letter_jobs")
|
||||
@statsd(namespace="tasks")
|
||||
def remove_letter_csv_files():
|
||||
_remove_csv_files([LETTER_TYPE])
|
||||
|
||||
|
||||
def _remove_csv_files(job_types):
|
||||
@@ -42,150 +62,151 @@ def _remove_csv_files(job_types):
|
||||
current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))
|
||||
|
||||
|
||||
@notify_celery.task(name="cleanup-unfinished-jobs")
|
||||
def cleanup_unfinished_jobs():
|
||||
now = utc_now()
|
||||
jobs = dao_get_unfinished_jobs()
|
||||
for job in jobs:
|
||||
# The query already checks that the processing_finished time is null, so here we are saying
|
||||
# if it started more than 4 hours ago, that's too long
|
||||
acceptable_finish_time = None
|
||||
try:
|
||||
if job.processing_started is not None:
|
||||
acceptable_finish_time = job.processing_started + timedelta(minutes=5)
|
||||
except TypeError:
|
||||
current_app.logger.exception(
|
||||
f"Job ID {job.id} processing_started is {job.processing_started}.",
|
||||
)
|
||||
raise
|
||||
if acceptable_finish_time and now > acceptable_finish_time:
|
||||
remove_csv_object(job.original_file_name)
|
||||
dao_archive_job(job)
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-notifications-older-than-retention")
|
||||
def delete_notifications_older_than_retention():
|
||||
delete_email_notifications_older_than_retention.apply_async(
|
||||
queue=QueueNames.REPORTING
|
||||
)
|
||||
delete_sms_notifications_older_than_retention.apply_async(
|
||||
queue=QueueNames.REPORTING
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-sms-notifications-older-than-retention")
|
||||
@notify_celery.task(name="delete-sms-notifications")
|
||||
@cronitor("delete-sms-notifications")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_sms_notifications_older_than_retention():
|
||||
_delete_notifications_older_than_retention_by_type(NotificationType.SMS)
|
||||
try:
|
||||
start = datetime.utcnow()
|
||||
deleted = delete_notifications_older_than_retention_by_type('sms')
|
||||
current_app.logger.info(
|
||||
"Delete {} job started {} finished {} deleted {} sms notifications".format(
|
||||
'sms',
|
||||
start,
|
||||
datetime.utcnow(),
|
||||
deleted
|
||||
)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete sms notifications")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-email-notifications-older-than-retention")
|
||||
@notify_celery.task(name="delete-email-notifications")
|
||||
@cronitor("delete-email-notifications")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_email_notifications_older_than_retention():
|
||||
_delete_notifications_older_than_retention_by_type(NotificationType.EMAIL)
|
||||
|
||||
|
||||
def _delete_notifications_older_than_retention_by_type(notification_type):
|
||||
flexible_data_retention = (
|
||||
fetch_service_data_retention_for_all_services_by_notification_type(
|
||||
notification_type
|
||||
try:
|
||||
start = datetime.utcnow()
|
||||
deleted = delete_notifications_older_than_retention_by_type('email')
|
||||
current_app.logger.info(
|
||||
"Delete {} job started {} finished {} deleted {} email notifications".format(
|
||||
'email',
|
||||
start,
|
||||
datetime.utcnow(),
|
||||
deleted
|
||||
)
|
||||
)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete email notifications")
|
||||
raise
|
||||
|
||||
for f in flexible_data_retention:
|
||||
day_to_delete_backwards_from = get_midnight_in_utc(
|
||||
utc_now()
|
||||
).date() - timedelta(days=f.days_of_retention)
|
||||
|
||||
delete_notifications_for_service_and_type.apply_async(
|
||||
queue=QueueNames.REPORTING,
|
||||
kwargs={
|
||||
"service_id": f.service_id,
|
||||
"notification_type": notification_type,
|
||||
"datetime_to_delete_before": day_to_delete_backwards_from,
|
||||
},
|
||||
@notify_celery.task(name="delete-letter-notifications")
|
||||
@cronitor("delete-letter-notifications")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_letter_notifications_older_than_retention():
|
||||
try:
|
||||
start = datetime.utcnow()
|
||||
deleted = delete_notifications_older_than_retention_by_type('letter')
|
||||
current_app.logger.info(
|
||||
"Delete {} job started {} finished {} deleted {} letter notifications".format(
|
||||
'letter',
|
||||
start,
|
||||
datetime.utcnow(),
|
||||
deleted
|
||||
)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete letter notifications")
|
||||
raise
|
||||
|
||||
seven_days_ago = get_midnight_in_utc(utc_now()).date() - timedelta(days=7)
|
||||
|
||||
service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
|
||||
@notify_celery.task(name='timeout-sending-notifications')
|
||||
@cronitor('timeout-sending-notifications')
|
||||
@statsd(namespace="tasks")
|
||||
def timeout_notifications():
|
||||
technical_failure_notifications, temporary_failure_notifications = \
|
||||
dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
|
||||
|
||||
# get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
|
||||
# This query takes a couple of mins to run.
|
||||
service_ids_that_have_sent_notifications_recently = (
|
||||
get_service_ids_with_notifications_before(notification_type, seven_days_ago)
|
||||
)
|
||||
|
||||
service_ids_to_purge = (
|
||||
service_ids_that_have_sent_notifications_recently
|
||||
- service_ids_with_data_retention
|
||||
)
|
||||
|
||||
for service_id in service_ids_to_purge:
|
||||
delete_notifications_for_service_and_type.apply_async(
|
||||
queue=QueueNames.REPORTING,
|
||||
kwargs={
|
||||
"service_id": service_id,
|
||||
"notification_type": notification_type,
|
||||
"datetime_to_delete_before": seven_days_ago,
|
||||
},
|
||||
)
|
||||
notifications = technical_failure_notifications + temporary_failure_notifications
|
||||
for notification in notifications:
|
||||
# queue callback task only if the service_callback_api exists
|
||||
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
|
||||
if service_callback_api:
|
||||
encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
|
||||
send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
|
||||
queue=QueueNames.CALLBACKS)
|
||||
|
||||
current_app.logger.info(
|
||||
f"delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: "
|
||||
f"{len(service_ids_with_data_retention)} services with flexible data retention, "
|
||||
f"{len(service_ids_to_purge)} services without flexible data retention"
|
||||
"Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
|
||||
if technical_failure_notifications:
|
||||
message = "{} notifications have been updated to technical-failure because they " \
|
||||
"have timed out and are still in created.Notification ids: {}".format(
|
||||
len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
|
||||
raise NotificationTechnicalFailureException(message)
|
||||
|
||||
|
||||
@notify_celery.task(name='send-daily-performance-platform-stats')
|
||||
@cronitor('send-daily-performance-platform-stats')
|
||||
@statsd(namespace="tasks")
|
||||
def send_daily_performance_platform_stats(date=None):
|
||||
# date is a string in the format of "YYYY-MM-DD"
|
||||
if date is None:
|
||||
date = (datetime.utcnow() - timedelta(days=1)).date()
|
||||
else:
|
||||
date = datetime.strptime(date, "%Y-%m-%d").date()
|
||||
|
||||
if performance_platform_client.active:
|
||||
|
||||
send_total_sent_notifications_to_performance_platform(bst_date=date)
|
||||
processing_time.send_processing_time_to_performance_platform(bst_date=date)
|
||||
|
||||
|
||||
def send_total_sent_notifications_to_performance_platform(bst_date):
|
||||
count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
|
||||
start_time = get_london_midnight_in_utc(bst_date)
|
||||
|
||||
email_sent_count = count_dict['email']
|
||||
sms_sent_count = count_dict['sms']
|
||||
letter_sent_count = count_dict['letter']
|
||||
|
||||
current_app.logger.info(
|
||||
"Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
|
||||
.format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-notifications-for-service-and-type")
|
||||
def delete_notifications_for_service_and_type(
|
||||
service_id, notification_type, datetime_to_delete_before
|
||||
):
|
||||
start = utc_now()
|
||||
num_deleted = move_notifications_to_notification_history(
|
||||
notification_type,
|
||||
service_id,
|
||||
datetime_to_delete_before,
|
||||
)
|
||||
if num_deleted:
|
||||
end = utc_now()
|
||||
current_app.logger.info(
|
||||
f"delete-notifications-for-service-and-type: "
|
||||
f"service: {service_id}, "
|
||||
f"notification_type: {notification_type}, "
|
||||
f"count deleted: {num_deleted}, "
|
||||
f"duration: {(end - start).seconds} seconds"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="timeout-sending-notifications")
|
||||
def timeout_notifications():
|
||||
notifications = ["dummy value so len() > 0"]
|
||||
|
||||
cutoff_time = utc_now() - timedelta(
|
||||
seconds=current_app.config.get("SENDING_NOTIFICATIONS_TIMEOUT_PERIOD")
|
||||
total_sent_notifications.send_total_notifications_sent_for_day_stats(
|
||||
start_time,
|
||||
'sms',
|
||||
sms_sent_count
|
||||
)
|
||||
|
||||
while len(notifications) > 0:
|
||||
notifications = dao_timeout_notifications(cutoff_time)
|
||||
total_sent_notifications.send_total_notifications_sent_for_day_stats(
|
||||
start_time,
|
||||
'email',
|
||||
email_sent_count
|
||||
)
|
||||
|
||||
for notification in notifications:
|
||||
check_and_queue_callback_task(notification)
|
||||
|
||||
current_app.logger.info(
|
||||
"Timeout period reached for {} notifications, status has been updated.".format(
|
||||
len(notifications)
|
||||
)
|
||||
)
|
||||
total_sent_notifications.send_total_notifications_sent_for_day_stats(
|
||||
start_time,
|
||||
'letter',
|
||||
letter_sent_count
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-inbound-sms")
|
||||
@cronitor("delete-inbound-sms")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_inbound_sms():
|
||||
try:
|
||||
start = utc_now()
|
||||
start = datetime.utcnow()
|
||||
deleted = delete_inbound_sms_older_than_retention()
|
||||
current_app.logger.info(
|
||||
"Delete inbound sms job started {} finished {} deleted {} inbound sms notifications".format(
|
||||
start, utc_now(), deleted
|
||||
start,
|
||||
datetime.utcnow(),
|
||||
deleted
|
||||
)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
@@ -193,22 +214,135 @@ def delete_inbound_sms():
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="save-daily-notification-processing-time")
|
||||
def save_daily_notification_processing_time(local_date=None):
|
||||
# local_date is a string in the format of "YYYY-MM-DD"
|
||||
if local_date is None:
|
||||
# if a date is not provided, we run against yesterdays data
|
||||
local_date = (utc_now() - timedelta(days=1)).date()
|
||||
else:
|
||||
local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
|
||||
@notify_celery.task(name="remove_transformed_dvla_files")
|
||||
@cronitor("remove_transformed_dvla_files")
|
||||
@statsd(namespace="tasks")
|
||||
def remove_transformed_dvla_files():
|
||||
jobs = dao_get_jobs_older_than_data_retention(notification_types=[LETTER_TYPE])
|
||||
for job in jobs:
|
||||
s3.remove_transformed_dvla_file(job.id)
|
||||
current_app.logger.info("Transformed dvla file for job {} has been removed from s3.".format(job.id))
|
||||
|
||||
start_time = get_midnight_in_utc(local_date)
|
||||
end_time = get_midnight_in_utc(local_date + timedelta(days=1))
|
||||
result = dao_get_notifications_processing_time_stats(start_time, end_time)
|
||||
insert_update_processing_time(
|
||||
FactProcessingTime(
|
||||
local_date=local_date,
|
||||
messages_total=result.messages_total,
|
||||
messages_within_10_secs=result.messages_within_10_secs,
|
||||
|
||||
# TODO: remove me, i'm not being run by anything
|
||||
@notify_celery.task(name="delete_dvla_response_files")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_dvla_response_files_older_than_seven_days():
|
||||
try:
|
||||
start = datetime.utcnow()
|
||||
bucket_objects = s3.get_s3_bucket_objects(
|
||||
current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
|
||||
'root/dispatch'
|
||||
)
|
||||
older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects)
|
||||
|
||||
for f in older_than_seven_days:
|
||||
s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key'])
|
||||
|
||||
current_app.logger.info(
|
||||
"Delete dvla response files started {} finished {} deleted {} files".format(
|
||||
start,
|
||||
datetime.utcnow(),
|
||||
len(older_than_seven_days)
|
||||
)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete dvla response files")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
|
||||
@cronitor("raise-alert-if-letter-notifications-still-sending")
|
||||
@statsd(namespace="tasks")
|
||||
def raise_alert_if_letter_notifications_still_sending():
|
||||
today = datetime.utcnow().date()
|
||||
|
||||
# Do nothing on the weekend
|
||||
if today.isoweekday() in {6, 7}: # sat, sun
|
||||
return
|
||||
|
||||
if today.isoweekday() in {1, 2}: # mon, tues. look for files from before the weekend
|
||||
offset_days = 4
|
||||
else:
|
||||
offset_days = 2
|
||||
|
||||
q = Notification.query.filter(
|
||||
Notification.notification_type == LETTER_TYPE,
|
||||
Notification.status == NOTIFICATION_SENDING,
|
||||
Notification.key_type == KEY_TYPE_NORMAL,
|
||||
func.date(Notification.sent_at) <= today - timedelta(days=offset_days)
|
||||
)
|
||||
|
||||
if today.isoweekday() in {2, 4}: # on tue, thu, we only care about first class letters
|
||||
q = q.filter(
|
||||
Notification.postage == 'first'
|
||||
)
|
||||
|
||||
still_sending = q.count()
|
||||
|
||||
if still_sending:
|
||||
message = "There are {} letters in the 'sending' state from {}".format(
|
||||
still_sending,
|
||||
(today - timedelta(days=offset_days)).strftime('%A %d %B')
|
||||
)
|
||||
# Only send alerts in production
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
|
||||
zendesk_client.create_ticket(
|
||||
subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
|
||||
message=message,
|
||||
ticket_type=zendesk_client.TYPE_INCIDENT
|
||||
)
|
||||
else:
|
||||
current_app.logger.info(message)
|
||||
|
||||
|
||||
@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
|
||||
@cronitor('raise-alert-if-no-letter-ack-file')
|
||||
@statsd(namespace="tasks")
|
||||
def letter_raise_alert_if_no_ack_file_for_zip():
|
||||
# get a list of zip files since yesterday
|
||||
zip_file_set = set()
|
||||
today_str = datetime.utcnow().strftime('%Y-%m-%d')
|
||||
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format
|
||||
|
||||
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
|
||||
subfolder=today_str + '/zips_sent',
|
||||
suffix='.TXT'):
|
||||
subname = key.split('/')[-1] # strip subfolder in name
|
||||
zip_file_set.add(subname.upper().replace('.ZIP.TXT', ''))
|
||||
|
||||
# get acknowledgement file
|
||||
ack_file_set = set()
|
||||
|
||||
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
|
||||
subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
|
||||
ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
|
||||
|
||||
message = (
|
||||
"Letter ack file does not contain all zip files sent. "
|
||||
"Missing ack for zip files: {}, "
|
||||
"pdf bucket: {}, subfolder: {}, "
|
||||
"ack bucket: {}"
|
||||
).format(
|
||||
str(sorted(zip_file_set - ack_file_set)),
|
||||
current_app.config['LETTERS_PDF_BUCKET_NAME'],
|
||||
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
|
||||
current_app.config['DVLA_RESPONSE_BUCKET_NAME']
|
||||
)
|
||||
# strip empty element before comparison
|
||||
ack_file_set.discard('')
|
||||
zip_file_set.discard('')
|
||||
|
||||
if len(zip_file_set - ack_file_set) > 0:
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
|
||||
zendesk_client.create_ticket(
|
||||
subject="Letter acknowledge error",
|
||||
message=message,
|
||||
ticket_type=zendesk_client.TYPE_INCIDENT
|
||||
)
|
||||
current_app.logger.error(message)
|
||||
|
||||
if len(ack_file_set - zip_file_set) > 0:
|
||||
current_app.logger.info(
|
||||
"letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
|
||||
)
|
||||
|
||||
@@ -1,276 +1,84 @@
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import iso8601
|
||||
from celery.exceptions import Retry
|
||||
from flask import current_app, json
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from app import notify_celery
|
||||
from app.celery.service_callback_tasks import (
|
||||
create_complaint_callback_data,
|
||||
create_delivery_status_callback_data,
|
||||
send_complaint_to_service,
|
||||
send_delivery_status_to_service,
|
||||
)
|
||||
from app.config import Config, QueueNames
|
||||
from app import notify_celery, statsd_client
|
||||
from app.config import QueueNames
|
||||
from app.clients.email.aws_ses import get_aws_responses
|
||||
from app.dao import notifications_dao
|
||||
from app.dao.complaint_dao import save_complaint
|
||||
from app.dao.notifications_dao import dao_get_notification_history_by_reference
|
||||
from app.dao.service_callback_api_dao import (
|
||||
get_service_complaint_callback_api_for_service,
|
||||
get_service_delivery_status_callback_api_for_service,
|
||||
from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
|
||||
|
||||
from app.notifications.notifications_ses_callback import (
|
||||
determine_notification_bounce_type,
|
||||
handle_complaint,
|
||||
_check_and_queue_complaint_callback_task,
|
||||
_check_and_queue_callback_task,
|
||||
)
|
||||
from app.enums import CallbackType, NotificationStatus
|
||||
from app.models import Complaint
|
||||
from app.utils import utc_now
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300
|
||||
)
|
||||
@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def process_ses_results(self, response):
|
||||
try:
|
||||
ses_message = json.loads(response["Message"])
|
||||
notification_type = ses_message["notificationType"]
|
||||
# TODO remove after smoke testing on prod is implemented
|
||||
current_app.logger.info(
|
||||
f"Attempting to process SES delivery status message "
|
||||
f"from SNS with type: {notification_type} and body: {ses_message}"
|
||||
)
|
||||
bounce_message = None
|
||||
ses_message = json.loads(response['Message'])
|
||||
notification_type = ses_message['notificationType']
|
||||
|
||||
if notification_type == "Bounce":
|
||||
bounce_message = determine_notification_bounce_type(ses_message)
|
||||
elif notification_type == "Complaint":
|
||||
if notification_type == 'Bounce':
|
||||
notification_type = determine_notification_bounce_type(notification_type, ses_message)
|
||||
elif notification_type == 'Complaint':
|
||||
_check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
|
||||
return True
|
||||
|
||||
aws_response_dict = get_aws_responses(ses_message)
|
||||
aws_response_dict = get_aws_responses(notification_type)
|
||||
|
||||
notification_status = aws_response_dict["notification_status"]
|
||||
reference = ses_message["mail"]["messageId"]
|
||||
notification_status = aws_response_dict['notification_status']
|
||||
reference = ses_message['mail']['messageId']
|
||||
|
||||
try:
|
||||
notification = notifications_dao.dao_get_notification_by_reference(
|
||||
reference
|
||||
)
|
||||
notification = notifications_dao.dao_get_notification_by_reference(reference)
|
||||
except NoResultFound:
|
||||
message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(
|
||||
tzinfo=None
|
||||
)
|
||||
if utc_now() - message_time < timedelta(minutes=5):
|
||||
current_app.logger.info(
|
||||
f"Notification not found for reference: {reference}"
|
||||
f"(while attempting update to {notification_status}). "
|
||||
f"Callback may have arrived before notification was"
|
||||
f"persisted to the DB. Adding task to retry queue"
|
||||
)
|
||||
self.retry(
|
||||
queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME
|
||||
)
|
||||
message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
|
||||
if datetime.utcnow() - message_time < timedelta(minutes=5):
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
else:
|
||||
current_app.logger.warning(
|
||||
f"Notification not found for reference: {reference} "
|
||||
f"(while attempting update to {notification_status})"
|
||||
"notification not found for reference: {} (update to {})".format(reference, notification_status)
|
||||
)
|
||||
return
|
||||
|
||||
if bounce_message:
|
||||
current_app.logger.info(
|
||||
f"SES bounce for notification ID {notification.id}: {bounce_message}"
|
||||
)
|
||||
|
||||
if notification.status not in {
|
||||
NotificationStatus.SENDING,
|
||||
NotificationStatus.PENDING,
|
||||
}:
|
||||
notifications_dao._duplicate_update_warning(
|
||||
notification, notification_status
|
||||
)
|
||||
if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
|
||||
notifications_dao._duplicate_update_warning(notification, notification_status)
|
||||
return
|
||||
|
||||
notifications_dao._update_notification_status(
|
||||
notification=notification,
|
||||
status=notification_status,
|
||||
provider_response=aws_response_dict["provider_response"],
|
||||
)
|
||||
notifications_dao._update_notification_status(notification=notification, status=notification_status)
|
||||
|
||||
if not aws_response_dict["success"]:
|
||||
if not aws_response_dict['success']:
|
||||
current_app.logger.info(
|
||||
"SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
|
||||
notification.id, reference, aws_response_dict["message"]
|
||||
notification.id, reference, aws_response_dict['message']
|
||||
)
|
||||
)
|
||||
else:
|
||||
current_app.logger.info(
|
||||
"SES callback return status of {} for notification: {}".format(
|
||||
notification_status, notification.id
|
||||
)
|
||||
)
|
||||
current_app.logger.info('SES callback return status of {} for notification: {}'.format(
|
||||
notification_status, notification.id
|
||||
))
|
||||
|
||||
check_and_queue_callback_task(notification)
|
||||
statsd_client.incr('callback.ses.{}'.format(notification_status))
|
||||
|
||||
if notification.sent_at:
|
||||
statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
|
||||
|
||||
_check_and_queue_callback_task(notification)
|
||||
|
||||
return True
|
||||
|
||||
except Retry:
|
||||
raise
|
||||
|
||||
except Exception:
|
||||
current_app.logger.exception("Error processing SES results")
|
||||
self.retry(queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME)
|
||||
|
||||
|
||||
def determine_notification_bounce_type(ses_message):
|
||||
notification_type = ses_message["notificationType"]
|
||||
if notification_type in ["Delivery", "Complaint"]:
|
||||
return notification_type
|
||||
|
||||
if notification_type != "Bounce":
|
||||
raise KeyError(f"Unhandled sns notification type {notification_type}")
|
||||
|
||||
remove_emails_from_bounce(ses_message)
|
||||
current_app.logger.info(
|
||||
"SES bounce dict: {}".format(
|
||||
json.dumps(ses_message).replace("{", "(").replace("}", ")")
|
||||
)
|
||||
)
|
||||
if ses_message["bounce"]["bounceType"] == "Permanent":
|
||||
return "Permanent"
|
||||
return "Temporary"
|
||||
|
||||
|
||||
def determine_notification_type(ses_message):
|
||||
notification_type = ses_message["notificationType"]
|
||||
if notification_type not in ["Bounce", "Complaint", "Delivery"]:
|
||||
raise KeyError(f"Unhandled sns notification type {notification_type}")
|
||||
if notification_type == "Bounce":
|
||||
return determine_notification_bounce_type(ses_message)
|
||||
return notification_type
|
||||
|
||||
|
||||
def _determine_provider_response(ses_message):
|
||||
if ses_message["notificationType"] != "Bounce":
|
||||
return None
|
||||
|
||||
bounce_type = ses_message["bounce"]["bounceType"]
|
||||
bounce_subtype = ses_message["bounce"]["bounceSubType"]
|
||||
|
||||
# See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
|
||||
if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
|
||||
return "The email address is on our email provider suppression list"
|
||||
elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
|
||||
return "The email address is on the GC Notify suppression list"
|
||||
elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
|
||||
return "The email was rejected because of its attachments"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_aws_responses(ses_message):
|
||||
status = determine_notification_type(ses_message)
|
||||
|
||||
base = {
|
||||
"Permanent": {
|
||||
"message": "Hard bounced",
|
||||
"success": False,
|
||||
"notification_status": NotificationStatus.PERMANENT_FAILURE,
|
||||
},
|
||||
"Temporary": {
|
||||
"message": "Soft bounced",
|
||||
"success": False,
|
||||
"notification_status": NotificationStatus.TEMPORARY_FAILURE,
|
||||
},
|
||||
"Delivery": {
|
||||
"message": "Delivered",
|
||||
"success": True,
|
||||
"notification_status": NotificationStatus.DELIVERED,
|
||||
},
|
||||
"Complaint": {
|
||||
"message": "Complaint",
|
||||
"success": True,
|
||||
"notification_status": NotificationStatus.DELIVERED,
|
||||
},
|
||||
}[status]
|
||||
|
||||
base["provider_response"] = _determine_provider_response(ses_message)
|
||||
|
||||
return base
|
||||
|
||||
|
||||
def handle_complaint(ses_message):
|
||||
recipient_email = remove_emails_from_complaint(ses_message)[0]
|
||||
current_app.logger.info(
|
||||
"Complaint from SES: \n{}".format(
|
||||
json.dumps(ses_message).replace("{", "(").replace("}", ")")
|
||||
)
|
||||
)
|
||||
try:
|
||||
reference = ses_message["mail"]["messageId"]
|
||||
except KeyError:
|
||||
current_app.logger.exception(
|
||||
"Complaint from SES failed to get reference from message"
|
||||
)
|
||||
return
|
||||
notification = dao_get_notification_history_by_reference(reference)
|
||||
ses_complaint = ses_message.get(CallbackType.COMPLAINT, None)
|
||||
|
||||
complaint = Complaint(
|
||||
notification_id=notification.id,
|
||||
service_id=notification.service_id,
|
||||
ses_feedback_id=(
|
||||
ses_complaint.get("feedbackId", None) if ses_complaint else None
|
||||
),
|
||||
complaint_type=(
|
||||
ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None
|
||||
),
|
||||
complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
|
||||
)
|
||||
save_complaint(complaint)
|
||||
return complaint, notification, recipient_email
|
||||
|
||||
|
||||
def remove_mail_headers(dict_to_edit):
|
||||
if dict_to_edit["mail"].get("headers"):
|
||||
dict_to_edit["mail"].pop("headers")
|
||||
if dict_to_edit["mail"].get("commonHeaders"):
|
||||
dict_to_edit["mail"].pop("commonHeaders")
|
||||
|
||||
|
||||
def remove_emails_from_bounce(bounce_dict):
|
||||
remove_mail_headers(bounce_dict)
|
||||
bounce_dict["mail"].pop("destination", None)
|
||||
bounce_dict["bounce"].pop("bouncedRecipients", None)
|
||||
|
||||
|
||||
def remove_emails_from_complaint(complaint_dict):
|
||||
remove_mail_headers(complaint_dict)
|
||||
complaint_dict[CallbackType.COMPLAINT].pop("complainedRecipients")
|
||||
return complaint_dict["mail"].pop("destination")
|
||||
|
||||
|
||||
def check_and_queue_callback_task(notification):
|
||||
# queue callback task only if the service_callback_api exists
|
||||
service_callback_api = get_service_delivery_status_callback_api_for_service(
|
||||
service_id=notification.service_id
|
||||
)
|
||||
if service_callback_api:
|
||||
notification_data = create_delivery_status_callback_data(
|
||||
notification, service_callback_api
|
||||
)
|
||||
send_delivery_status_to_service.apply_async(
|
||||
[str(notification.id), notification_data], queue=QueueNames.CALLBACKS
|
||||
)
|
||||
|
||||
|
||||
def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
|
||||
# queue callback task only if the service_callback_api exists
|
||||
service_callback_api = get_service_complaint_callback_api_for_service(
|
||||
service_id=notification.service_id
|
||||
)
|
||||
if service_callback_api:
|
||||
complaint_data = create_complaint_callback_data(
|
||||
complaint, notification, service_callback_api, recipient
|
||||
)
|
||||
send_complaint_to_service.apply_async(
|
||||
[complaint_data], queue=QueueNames.CALLBACKS
|
||||
)
|
||||
except Exception as e:
|
||||
current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
|
||||
@@ -1,134 +1,63 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from flask import current_app
|
||||
from notifications_utils.recipients import InvalidEmailError
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from app import notify_celery, redis_store
|
||||
from app.clients.email import EmailClientNonRetryableException
|
||||
from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
|
||||
from app.clients.sms import SmsClientResponseException
|
||||
from app.config import Config, QueueNames
|
||||
from app import notify_celery
|
||||
from app.config import QueueNames
|
||||
from app.dao import notifications_dao
|
||||
from app.dao.notifications_dao import update_notification_status_by_id
|
||||
from app.delivery import send_to_providers
|
||||
from app.enums import NotificationStatus
|
||||
from app.exceptions import NotificationTechnicalFailureException
|
||||
from notifications_utils.clients.redis import total_limit_cache_key
|
||||
from app.models import NOTIFICATION_TECHNICAL_FAILURE
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300
|
||||
)
|
||||
@notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def deliver_sms(self, notification_id):
|
||||
"""Branch off to the final step in delivering the notification to sns and get delivery receipts."""
|
||||
try:
|
||||
current_app.logger.info(
|
||||
"Start sending SMS for notification id: {}".format(notification_id)
|
||||
)
|
||||
current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
|
||||
notification = notifications_dao.get_notification_by_id(notification_id)
|
||||
ansi_green = "\033[32m"
|
||||
ansi_reset = "\033[0m"
|
||||
|
||||
if not notification:
|
||||
raise NoResultFound()
|
||||
if (
|
||||
os.getenv("NOTIFY_ENVIRONMENT") == "development"
|
||||
and "authentication code" in notification.content
|
||||
):
|
||||
current_app.logger.warning(
|
||||
ansi_green + f"AUTHENTICATION CODE: {notification.content}" + ansi_reset
|
||||
)
|
||||
# Code branches off to send_to_providers.py
|
||||
send_to_providers.send_sms_to_provider(notification)
|
||||
|
||||
cache_key = total_limit_cache_key(notification.service_id)
|
||||
redis_store.incr(cache_key)
|
||||
|
||||
except Exception as e:
|
||||
update_notification_status_by_id(
|
||||
notification_id,
|
||||
NotificationStatus.TEMPORARY_FAILURE,
|
||||
)
|
||||
if isinstance(e, SmsClientResponseException):
|
||||
current_app.logger.warning(
|
||||
"SMS notification delivery for id: {} failed".format(notification_id),
|
||||
)
|
||||
else:
|
||||
current_app.logger.exception(
|
||||
"SMS notification delivery for id: {} failed".format(notification_id),
|
||||
)
|
||||
|
||||
except Exception:
|
||||
try:
|
||||
current_app.logger.exception(
|
||||
"SMS notification delivery for id: {} failed".format(notification_id)
|
||||
)
|
||||
if self.request.retries == 0:
|
||||
self.retry(
|
||||
queue=QueueNames.RETRY,
|
||||
countdown=0,
|
||||
expires=Config.DEFAULT_REDIS_EXPIRE_TIME,
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY, countdown=0)
|
||||
else:
|
||||
self.retry(
|
||||
queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
message = (
|
||||
"RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. "
|
||||
"Notification has been updated to technical-failure".format(
|
||||
notification_id
|
||||
)
|
||||
)
|
||||
update_notification_status_by_id(
|
||||
notification_id,
|
||||
NotificationStatus.TECHNICAL_FAILURE,
|
||||
)
|
||||
message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \
|
||||
"Notification has been updated to technical-failure".format(notification_id)
|
||||
update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
raise NotificationTechnicalFailureException(message)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="deliver_email", max_retries=48, default_retry_delay=30
|
||||
)
|
||||
@notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def deliver_email(self, notification_id):
|
||||
try:
|
||||
current_app.logger.info(
|
||||
"Start sending email for notification id: {}".format(notification_id)
|
||||
)
|
||||
current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
|
||||
notification = notifications_dao.get_notification_by_id(notification_id)
|
||||
|
||||
if not notification:
|
||||
raise NoResultFound()
|
||||
personalisation = redis_store.get(f"email-personalisation-{notification_id}")
|
||||
recipient = redis_store.get(f"email-recipient-{notification_id}")
|
||||
if personalisation:
|
||||
notification.personalisation = json.loads(personalisation)
|
||||
if recipient:
|
||||
notification.recipient = json.loads(recipient)
|
||||
|
||||
send_to_providers.send_email_to_provider(notification)
|
||||
except EmailClientNonRetryableException:
|
||||
current_app.logger.exception(f"Email notification {notification_id} failed")
|
||||
update_notification_status_by_id(notification_id, "technical-failure")
|
||||
except Exception as e:
|
||||
except InvalidEmailError as e:
|
||||
current_app.logger.exception(e)
|
||||
update_notification_status_by_id(notification_id, 'technical-failure')
|
||||
except Exception:
|
||||
try:
|
||||
if isinstance(e, AwsSesClientThrottlingSendRateException):
|
||||
current_app.logger.warning(
|
||||
f"RETRY: Email notification {notification_id} was rate limited by SES"
|
||||
)
|
||||
else:
|
||||
current_app.logger.exception(
|
||||
f"RETRY: Email notification {notification_id} failed"
|
||||
)
|
||||
|
||||
self.retry(queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME)
|
||||
current_app.logger.exception(
|
||||
"RETRY: Email notification {} failed".format(notification_id)
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
message = (
|
||||
"RETRY FAILED: Max retries reached. "
|
||||
"The task send_email_to_provider failed for notification {}. "
|
||||
"Notification has been updated to technical-failure".format(
|
||||
notification_id
|
||||
)
|
||||
)
|
||||
update_notification_status_by_id(
|
||||
notification_id,
|
||||
NotificationStatus.TECHNICAL_FAILURE,
|
||||
)
|
||||
message = "RETRY FAILED: Max retries reached. " \
|
||||
"The task send_email_to_provider failed for notification {}. " \
|
||||
"Notification has been updated to technical-failure".format(notification_id)
|
||||
update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
|
||||
raise NotificationTechnicalFailureException(message)
|
||||
|
||||
@@ -1,120 +1,102 @@
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from flask import current_app
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from notifications_utils.timezones import convert_utc_to_bst
|
||||
|
||||
from app import notify_celery
|
||||
from app.config import QueueNames
|
||||
from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
|
||||
from app.dao.fact_notification_status_dao import update_fact_notification_status
|
||||
from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
|
||||
from app.enums import NotificationType
|
||||
from app.utils import utc_now
|
||||
from app.cronitor import cronitor
|
||||
from app.dao.fact_billing_dao import (
|
||||
fetch_billing_data_for_day,
|
||||
update_fact_billing
|
||||
)
|
||||
from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
|
||||
from app.models import (
|
||||
SMS_TYPE,
|
||||
EMAIL_TYPE,
|
||||
LETTER_TYPE,
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="create-nightly-billing")
|
||||
@cronitor("create-nightly-billing")
|
||||
@statsd(namespace="tasks")
|
||||
def create_nightly_billing(day_start=None):
|
||||
# day_start is a datetime.date() object. e.g.
|
||||
# up to 4 days of data counting back from day_start is consolidated
|
||||
if day_start is None:
|
||||
day_start = utc_now().date() - timedelta(days=1)
|
||||
day_start = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
|
||||
else:
|
||||
# When calling the task its a string in the format of "YYYY-MM-DD"
|
||||
day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
|
||||
for i in range(0, 10):
|
||||
process_day = (day_start - timedelta(days=i)).isoformat()
|
||||
for i in range(0, 4):
|
||||
process_day = day_start - timedelta(days=i)
|
||||
|
||||
create_nightly_billing_for_day.apply_async(
|
||||
kwargs={"process_day": process_day}, queue=QueueNames.REPORTING
|
||||
)
|
||||
current_app.logger.info(
|
||||
f"create-nightly-billing task: create-nightly-billing-for-day task created for {process_day}"
|
||||
kwargs={'process_day': process_day.isoformat()},
|
||||
queue=QueueNames.REPORTING
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="create-nightly-billing-for-day")
|
||||
@statsd(namespace="tasks")
|
||||
def create_nightly_billing_for_day(process_day):
|
||||
process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
|
||||
current_app.logger.info(
|
||||
f"create-nightly-billing-for-day task for {process_day}: started"
|
||||
)
|
||||
|
||||
start = utc_now()
|
||||
start = datetime.utcnow()
|
||||
transit_data = fetch_billing_data_for_day(process_day=process_day)
|
||||
end = utc_now()
|
||||
end = datetime.utcnow()
|
||||
|
||||
current_app.logger.info(
|
||||
f"create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds"
|
||||
)
|
||||
current_app.logger.info(f'create-nightly-billing-for-day {process_day} fetched in {(end - start).seconds} seconds')
|
||||
|
||||
for data in transit_data:
|
||||
update_fact_billing(data, process_day)
|
||||
|
||||
current_app.logger.info(
|
||||
f"create-nightly-billing-for-day task for {process_day}: "
|
||||
f"task complete. {len(transit_data)} rows updated"
|
||||
f"create-nightly-billing-for-day task complete. {len(transit_data)} rows updated for day: {process_day}"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="create-nightly-notification-status")
|
||||
@cronitor("create-nightly-notification-status")
|
||||
@statsd(namespace="tasks")
|
||||
def create_nightly_notification_status():
|
||||
"""
|
||||
Aggregate notification statuses into rows in ft_notification_status.
|
||||
In order to minimise effort, this task assumes that:
|
||||
yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
|
||||
|
||||
- Email + SMS statuses don't change after 3 days. This is currently true
|
||||
because all outstanding email / SMS are "timed out" after 3 days, and
|
||||
we reject delivery receipts after this point.
|
||||
|
||||
Because the time range of the task exceeds the minimum possible retention
|
||||
period (3 days), we need to choose which table to query for each service.
|
||||
|
||||
The aggregation happens for 1 extra day in case:
|
||||
|
||||
- This task or the "timeout" task fails to run.
|
||||
|
||||
- Data is (somehow) still in transit to the history table, which would
|
||||
mean the aggregated results are temporarily incorrect.
|
||||
"""
|
||||
|
||||
yesterday = utc_now().date() - timedelta(days=1)
|
||||
|
||||
for notification_type in (NotificationType.SMS, NotificationType.EMAIL):
|
||||
days = 4
|
||||
|
||||
for i in range(days):
|
||||
process_day = yesterday - timedelta(days=i)
|
||||
|
||||
relevant_service_ids = get_service_ids_with_notifications_on_date(
|
||||
notification_type, process_day
|
||||
# email and sms
|
||||
for i in range(4):
|
||||
process_day = yesterday - timedelta(days=i)
|
||||
for notification_type in [SMS_TYPE, EMAIL_TYPE]:
|
||||
create_nightly_notification_status_for_day.apply_async(
|
||||
kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
|
||||
queue=QueueNames.REPORTING
|
||||
)
|
||||
|
||||
for service_id in relevant_service_ids:
|
||||
create_nightly_notification_status_for_service_and_day.apply_async(
|
||||
kwargs={
|
||||
"process_day": process_day.isoformat(),
|
||||
"notification_type": notification_type,
|
||||
"service_id": service_id,
|
||||
},
|
||||
queue=QueueNames.REPORTING,
|
||||
)
|
||||
# letters get modified for a longer time period than sms and email, so we need to reprocess for more days
|
||||
for i in range(10):
|
||||
process_day = yesterday - timedelta(days=i)
|
||||
create_nightly_notification_status_for_day.apply_async(
|
||||
kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
|
||||
queue=QueueNames.REPORTING
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
|
||||
def create_nightly_notification_status_for_service_and_day(
|
||||
process_day, service_id, notification_type
|
||||
):
|
||||
@notify_celery.task(name="create-nightly-notification-status-for-day")
|
||||
@statsd(namespace="tasks")
|
||||
def create_nightly_notification_status_for_day(process_day, notification_type):
|
||||
process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
|
||||
|
||||
start = utc_now()
|
||||
update_fact_notification_status(
|
||||
process_day=process_day,
|
||||
notification_type=notification_type,
|
||||
service_id=service_id,
|
||||
start = datetime.utcnow()
|
||||
transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
|
||||
end = datetime.utcnow()
|
||||
current_app.logger.info(
|
||||
f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
|
||||
f'data fetched in {(end - start).seconds} seconds'
|
||||
)
|
||||
|
||||
end = utc_now()
|
||||
update_fact_notification_status(transit_data, process_day, notification_type)
|
||||
|
||||
current_app.logger.info(
|
||||
f"create-nightly-notification-status-for-service-and-day task update "
|
||||
f"for {service_id}, {notification_type} for {process_day}: "
|
||||
f"updated in {(end - start).seconds} seconds"
|
||||
f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
|
||||
f'task complete - {len(transit_data)} rows updated'
|
||||
)
|
||||
|
||||
@@ -1,27 +1,46 @@
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
|
||||
from flask import current_app
|
||||
from requests import HTTPError, request
|
||||
from requests import request, RequestException, HTTPError
|
||||
|
||||
from app.celery.process_ses_receipts_tasks import process_ses_results
|
||||
from notifications_utils.s3 import s3upload
|
||||
|
||||
from app import notify_celery
|
||||
from app.aws.s3 import file_exists
|
||||
from app.models import SMS_TYPE
|
||||
from app.config import QueueNames
|
||||
from app.dao.notifications_dao import get_notification_by_id
|
||||
from app.enums import NotificationType
|
||||
from app.celery.process_ses_receipts_tasks import process_ses_results
|
||||
|
||||
temp_fail = "2028675303"
|
||||
perm_fail = "2028675302"
|
||||
delivered = "2028675309"
|
||||
temp_fail = "7700900003"
|
||||
perm_fail = "7700900002"
|
||||
delivered = "7700900001"
|
||||
|
||||
delivered_email = "delivered@simulator.notify"
|
||||
perm_fail_email = "perm-fail@simulator.notify"
|
||||
temp_fail_email = "temp-fail@simulator.notify"
|
||||
|
||||
|
||||
def send_sms_response(provider, reference):
|
||||
body = sns_callback(reference)
|
||||
headers = {"Content-type": "application/json"}
|
||||
def send_sms_response(provider, reference, to):
|
||||
if provider == "mmg":
|
||||
body = mmg_callback(reference, to)
|
||||
headers = {"Content-type": "application/json"}
|
||||
else:
|
||||
headers = {"Content-type": "application/x-www-form-urlencoded"}
|
||||
body = firetext_callback(reference, to)
|
||||
# to simulate getting a temporary_failure from firetext
|
||||
# we need to send a pending status updated then a permanent-failure
|
||||
if body['status'] == '2': # pending status
|
||||
make_request(SMS_TYPE, provider, body, headers)
|
||||
# 1 is a declined status for firetext, will result in a temp-failure
|
||||
body = {'mobile': to,
|
||||
'status': "1",
|
||||
'time': '2016-03-10 14:17:00',
|
||||
'reference': reference
|
||||
}
|
||||
|
||||
make_request(NotificationType.SMS, provider, body, headers)
|
||||
make_request(SMS_TYPE, provider, body, headers)
|
||||
|
||||
|
||||
def send_email_response(reference, to):
|
||||
@@ -36,155 +55,250 @@ def send_email_response(reference, to):
|
||||
|
||||
|
||||
def make_request(notification_type, provider, data, headers):
|
||||
api_call = "{}/notifications/{}/{}".format(
|
||||
current_app.config["API_HOST_NAME"], notification_type, provider
|
||||
)
|
||||
api_call = "{}/notifications/{}/{}".format(current_app.config["API_HOST_NAME"], notification_type, provider)
|
||||
|
||||
try:
|
||||
response = request("POST", api_call, headers=headers, data=data, timeout=60)
|
||||
response = request(
|
||||
"POST",
|
||||
api_call,
|
||||
headers=headers,
|
||||
data=data,
|
||||
timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
except HTTPError as e:
|
||||
except RequestException as e:
|
||||
api_error = HTTPError(e)
|
||||
current_app.logger.error(
|
||||
"API POST request on {} failed with status {}".format(
|
||||
api_call, e.response.status_code
|
||||
"API {} request on {} failed with {}".format(
|
||||
"POST",
|
||||
api_call,
|
||||
api_error.response
|
||||
)
|
||||
)
|
||||
raise e
|
||||
raise api_error
|
||||
finally:
|
||||
current_app.logger.info("Mocked provider callback request finished")
|
||||
return response.json()
|
||||
|
||||
|
||||
def sns_callback(notification_id):
|
||||
notification = get_notification_by_id(notification_id)
|
||||
def mmg_callback(notification_id, to):
|
||||
"""
|
||||
status: 3 - delivered
|
||||
status: 4 - expired (temp failure)
|
||||
status: 5 - rejected (perm failure)
|
||||
"""
|
||||
|
||||
# This will only work if all notifications, including successful ones, are in the notifications table
|
||||
# If we decide to delete successful notifications, we will have to get this from notifications history
|
||||
return json.dumps(
|
||||
{
|
||||
"CID": str(notification_id),
|
||||
"status": notification.status,
|
||||
# "deliverytime": notification.completed_at
|
||||
}
|
||||
if to.strip().endswith(temp_fail):
|
||||
status = "4"
|
||||
elif to.strip().endswith(perm_fail):
|
||||
status = "5"
|
||||
else:
|
||||
status = "3"
|
||||
|
||||
return json.dumps({"reference": "mmg_reference",
|
||||
"CID": str(notification_id),
|
||||
"MSISDN": to,
|
||||
"status": status,
|
||||
"deliverytime": "2016-04-05 16:01:07"})
|
||||
|
||||
|
||||
def firetext_callback(notification_id, to):
|
||||
"""
|
||||
status: 0 - delivered
|
||||
status: 1 - perm failure
|
||||
"""
|
||||
if to.strip().endswith(perm_fail):
|
||||
status = "1"
|
||||
elif to.strip().endswith(temp_fail):
|
||||
status = "2"
|
||||
else:
|
||||
status = "0"
|
||||
return {
|
||||
'mobile': to,
|
||||
'status': status,
|
||||
'time': '2016-03-10 14:17:00',
|
||||
'reference': notification_id
|
||||
}
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="create-fake-letter-response-file", max_retries=5, default_retry_delay=300)
|
||||
def create_fake_letter_response_file(self, reference):
|
||||
now = datetime.utcnow()
|
||||
dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
|
||||
|
||||
# try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
|
||||
for i in sorted(range(30), key=lambda _: random.random()):
|
||||
upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
|
||||
if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
|
||||
break
|
||||
else:
|
||||
raise ValueError(
|
||||
'cant create fake letter response file for {} - too many files for that time already exist on s3'.format(
|
||||
reference
|
||||
)
|
||||
)
|
||||
|
||||
s3upload(
|
||||
filedata=dvla_response_data,
|
||||
region=current_app.config['AWS_REGION'],
|
||||
bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
|
||||
file_location=upload_file_name
|
||||
)
|
||||
current_app.logger.info("Fake DVLA response file {}, content [{}], uploaded to {}, created at {}".format(
|
||||
upload_file_name, dvla_response_data, current_app.config['DVLA_RESPONSE_BUCKET_NAME'], now))
|
||||
|
||||
# on development we can't trigger SNS callbacks so we need to manually hit the DVLA callback endpoint
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] == 'development':
|
||||
make_request('letter', 'dvla', _fake_sns_s3_callback(upload_file_name), None)
|
||||
|
||||
|
||||
def _fake_sns_s3_callback(filename):
|
||||
message_contents = '{"Records":[{"s3":{"object":{"key":"%s"}}}]}' % (filename) # noqa
|
||||
return json.dumps({
|
||||
"Type": "Notification",
|
||||
"MessageId": "some-message-id",
|
||||
"Message": message_contents
|
||||
})
|
||||
|
||||
|
||||
def ses_notification_callback(reference):
|
||||
ses_message_body = {
|
||||
"delivery": {
|
||||
"processingTimeMillis": 2003,
|
||||
"recipients": ["success@simulator.amazonses.com"],
|
||||
"remoteMtaIp": "123.123.123.123",
|
||||
"reportingMTA": "a7-32.smtp-out.us-west-2.amazonses.com",
|
||||
"smtpResponse": "250 2.6.0 Message received",
|
||||
"timestamp": "2017-11-17T12:14:03.646Z",
|
||||
'delivery': {
|
||||
'processingTimeMillis': 2003,
|
||||
'recipients': ['success@simulator.amazonses.com'],
|
||||
'remoteMtaIp': '123.123.123.123',
|
||||
'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
|
||||
'smtpResponse': '250 2.6.0 Message received',
|
||||
'timestamp': '2017-11-17T12:14:03.646Z'
|
||||
},
|
||||
"mail": {
|
||||
"commonHeaders": {
|
||||
"from": ["TEST <TEST@notify.works>"],
|
||||
"subject": "lambda test",
|
||||
"to": ["success@simulator.amazonses.com"],
|
||||
'mail': {
|
||||
'commonHeaders': {
|
||||
'from': ['TEST <TEST@notify.works>'],
|
||||
'subject': 'lambda test',
|
||||
'to': ['success@simulator.amazonses.com']
|
||||
},
|
||||
"destination": ["success@simulator.amazonses.com"],
|
||||
"headers": [
|
||||
{"name": "From", "value": "TEST <TEST@notify.works>"},
|
||||
{"name": "To", "value": "success@simulator.amazonses.com"},
|
||||
{"name": "Subject", "value": "lambda test"},
|
||||
{"name": "MIME-Version", "value": "1.0"},
|
||||
'destination': ['success@simulator.amazonses.com'],
|
||||
'headers': [
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"',
|
||||
'name': 'From',
|
||||
'value': 'TEST <TEST@notify.works>'
|
||||
},
|
||||
{
|
||||
'name': 'To',
|
||||
'value': 'success@simulator.amazonses.com'
|
||||
},
|
||||
{
|
||||
'name': 'Subject',
|
||||
'value': 'lambda test'
|
||||
},
|
||||
{
|
||||
'name': 'MIME-Version',
|
||||
'value': '1.0'
|
||||
},
|
||||
{
|
||||
'name': 'Content-Type',
|
||||
'value': 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"'
|
||||
}
|
||||
],
|
||||
"headersTruncated": False,
|
||||
"messageId": reference,
|
||||
"sendingAccountId": "12341234",
|
||||
"source": '"TEST" <TEST@notify.works>',
|
||||
"sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
|
||||
"sourceIp": "0.0.0.1",
|
||||
"timestamp": "2017-11-17T12:14:01.643Z",
|
||||
'headersTruncated': False,
|
||||
'messageId': reference,
|
||||
'sendingAccountId': '12341234',
|
||||
'source': '"TEST" <TEST@notify.works>',
|
||||
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
|
||||
'sourceIp': '0.0.0.1',
|
||||
'timestamp': '2017-11-17T12:14:01.643Z'
|
||||
},
|
||||
"notificationType": "Delivery",
|
||||
'notificationType': 'Delivery'
|
||||
}
|
||||
|
||||
return {
|
||||
"Type": "Notification",
|
||||
"MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
|
||||
"TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
|
||||
"Subject": None,
|
||||
"Message": json.dumps(ses_message_body),
|
||||
"Timestamp": "2017-11-17T12:14:03.710Z",
|
||||
"SignatureVersion": "1",
|
||||
"Signature": "[REDACTED]",
|
||||
"SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem",
|
||||
"UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]",
|
||||
"MessageAttributes": {},
|
||||
'Type': 'Notification',
|
||||
'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
|
||||
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
|
||||
'Subject': None,
|
||||
'Message': json.dumps(ses_message_body),
|
||||
'Timestamp': '2017-11-17T12:14:03.710Z',
|
||||
'SignatureVersion': '1',
|
||||
'Signature': '[REDACTED]',
|
||||
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
|
||||
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
|
||||
'MessageAttributes': {}
|
||||
}
|
||||
|
||||
|
||||
def ses_hard_bounce_callback(reference):
|
||||
return _ses_bounce_callback(reference, "Permanent")
|
||||
return _ses_bounce_callback(reference, 'Permanent')
|
||||
|
||||
|
||||
def ses_soft_bounce_callback(reference):
|
||||
return _ses_bounce_callback(reference, "Temporary")
|
||||
return _ses_bounce_callback(reference, 'Temporary')
|
||||
|
||||
|
||||
def _ses_bounce_callback(reference, bounce_type):
|
||||
ses_message_body = {
|
||||
"bounce": {
|
||||
"bounceSubType": "General",
|
||||
"bounceType": bounce_type,
|
||||
"bouncedRecipients": [
|
||||
'bounce': {
|
||||
'bounceSubType': 'General',
|
||||
'bounceType': bounce_type,
|
||||
'bouncedRecipients': [{
|
||||
'action': 'failed',
|
||||
'diagnosticCode': 'smtp; 550 5.1.1 user unknown',
|
||||
'emailAddress': 'bounce@simulator.amazonses.com',
|
||||
'status': '5.1.1'
|
||||
}],
|
||||
'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
|
||||
'remoteMtaIp': '123.123.123.123',
|
||||
'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
|
||||
'timestamp': '2017-11-17T12:14:05.131Z'
|
||||
},
|
||||
'mail': {
|
||||
'commonHeaders': {
|
||||
'from': ['TEST <TEST@notify.works>'],
|
||||
'subject': 'ses callback test',
|
||||
'to': ['bounce@simulator.amazonses.com']
|
||||
},
|
||||
'destination': ['bounce@simulator.amazonses.com'],
|
||||
'headers': [
|
||||
{
|
||||
"action": "failed",
|
||||
"diagnosticCode": "smtp; 550 5.1.1 user unknown",
|
||||
"emailAddress": "bounce@simulator.amazonses.com",
|
||||
"status": "5.1.1",
|
||||
'name': 'From',
|
||||
'value': 'TEST <TEST@notify.works>'
|
||||
},
|
||||
{
|
||||
'name': 'To',
|
||||
'value': 'bounce@simulator.amazonses.com'
|
||||
},
|
||||
{
|
||||
'name': 'Subject',
|
||||
'value': 'lambda test'
|
||||
},
|
||||
{
|
||||
'name': 'MIME-Version',
|
||||
'value': '1.0'
|
||||
},
|
||||
{
|
||||
'name': 'Content-Type',
|
||||
'value': 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"'
|
||||
}
|
||||
],
|
||||
"feedbackId": "0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000",
|
||||
"remoteMtaIp": "123.123.123.123",
|
||||
"reportingMTA": "dsn; a7-31.smtp-out.us-west-2.amazonses.com",
|
||||
"timestamp": "2017-11-17T12:14:05.131Z",
|
||||
'headersTruncated': False,
|
||||
'messageId': reference,
|
||||
'sendingAccountId': '12341234',
|
||||
'source': '"TEST" <TEST@notify.works>',
|
||||
'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
|
||||
'sourceIp': '0.0.0.1',
|
||||
'timestamp': '2017-11-17T12:14:03.000Z'
|
||||
},
|
||||
"mail": {
|
||||
"commonHeaders": {
|
||||
"from": ["TEST <TEST@notify.works>"],
|
||||
"subject": "ses callback test",
|
||||
"to": ["bounce@simulator.amazonses.com"],
|
||||
},
|
||||
"destination": ["bounce@simulator.amazonses.com"],
|
||||
"headers": [
|
||||
{"name": "From", "value": "TEST <TEST@notify.works>"},
|
||||
{"name": "To", "value": "bounce@simulator.amazonses.com"},
|
||||
{"name": "Subject", "value": "lambda test"},
|
||||
{"name": "MIME-Version", "value": "1.0"},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"',
|
||||
},
|
||||
],
|
||||
"headersTruncated": False,
|
||||
"messageId": reference,
|
||||
"sendingAccountId": "12341234",
|
||||
"source": '"TEST" <TEST@notify.works>',
|
||||
"sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
|
||||
"sourceIp": "0.0.0.1",
|
||||
"timestamp": "2017-11-17T12:14:03.000Z",
|
||||
},
|
||||
"notificationType": "Bounce",
|
||||
'notificationType': 'Bounce'
|
||||
}
|
||||
return {
|
||||
"Type": "Notification",
|
||||
"MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
|
||||
"TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
|
||||
"Subject": None,
|
||||
"Message": json.dumps(ses_message_body),
|
||||
"Timestamp": "2017-11-17T12:14:05.149Z",
|
||||
"SignatureVersion": "1",
|
||||
"Signature": "[REDACTED]", # noqa
|
||||
"SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem",
|
||||
"UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]",
|
||||
"MessageAttributes": {},
|
||||
'Type': 'Notification',
|
||||
'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
|
||||
'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
|
||||
'Subject': None,
|
||||
'Message': json.dumps(ses_message_body),
|
||||
'Timestamp': '2017-11-17T12:14:05.149Z',
|
||||
'SignatureVersion': '1',
|
||||
'Signature': '[REDACTED]', # noqa
|
||||
'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
|
||||
'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
|
||||
'MessageAttributes': {}
|
||||
}
|
||||
|
||||
@@ -1,208 +1,279 @@
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import (
|
||||
datetime,
|
||||
timedelta
|
||||
)
|
||||
|
||||
from flask import current_app
|
||||
from sqlalchemy import between, select, union
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from app import db, notify_celery, redis_store, zendesk_client
|
||||
from app import notify_celery, zendesk_client
|
||||
from app.celery.tasks import (
|
||||
get_recipient_csv_and_template_and_sender_id,
|
||||
process_incomplete_jobs,
|
||||
process_job,
|
||||
process_row,
|
||||
get_recipient_csv_and_template_and_sender_id,
|
||||
process_row
|
||||
)
|
||||
from app.clients.cloudwatch.aws_cloudwatch import AwsCloudwatchClient
|
||||
from app.config import QueueNames
|
||||
from app.dao.invited_org_user_dao import (
|
||||
delete_org_invitations_created_more_than_two_days_ago,
|
||||
)
|
||||
from app.dao.invited_user_dao import expire_invitations_created_more_than_two_days_ago
|
||||
from app.celery.letters_pdf_tasks import create_letters_pdf
|
||||
from app.config import QueueNames, TaskNames
|
||||
from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
|
||||
from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
|
||||
from app.dao.jobs_dao import (
|
||||
dao_set_scheduled_jobs_to_pending,
|
||||
dao_update_job_status_to_error,
|
||||
find_jobs_with_missing_rows,
|
||||
find_missing_row_for_job,
|
||||
find_missing_row_for_job
|
||||
)
|
||||
from app.dao.jobs_dao import dao_update_job
|
||||
from app.dao.notifications_dao import (
|
||||
dao_batch_insert_notifications,
|
||||
dao_close_out_delivery_receipts,
|
||||
dao_update_delivery_receipts,
|
||||
dao_get_scheduled_notifications,
|
||||
set_scheduled_notification_to_processed,
|
||||
notifications_not_yet_sent,
|
||||
dao_precompiled_letters_still_pending_virus_check,
|
||||
dao_old_letters_with_created_status,
|
||||
letters_missing_from_sending_bucket,
|
||||
is_delivery_slow_for_providers,
|
||||
)
|
||||
from app.dao.services_dao import (
|
||||
dao_find_services_sending_to_tv_numbers,
|
||||
dao_find_services_with_high_failure_rates,
|
||||
from app.dao.provider_details_dao import (
|
||||
dao_reduce_sms_provider_priority,
|
||||
dao_adjust_provider_priority_back_to_resting_points
|
||||
)
|
||||
from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
|
||||
from app.enums import JobStatus, NotificationType
|
||||
from app.models import Job, Notification
|
||||
from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
|
||||
from app.models import (
|
||||
Job,
|
||||
JOB_STATUS_IN_PROGRESS,
|
||||
JOB_STATUS_ERROR,
|
||||
SMS_TYPE,
|
||||
EMAIL_TYPE,
|
||||
)
|
||||
from app.notifications.process_notifications import send_notification_to_queue
|
||||
from app.utils import utc_now
|
||||
from notifications_utils import aware_utcnow
|
||||
from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTicket
|
||||
|
||||
MAX_NOTIFICATION_FAILS = 10000
|
||||
from app.v2.errors import JobIncompleteError
|
||||
|
||||
|
||||
@notify_celery.task(name="run-scheduled-jobs")
|
||||
@statsd(namespace="tasks")
|
||||
def run_scheduled_jobs():
|
||||
try:
|
||||
for job in dao_set_scheduled_jobs_to_pending():
|
||||
process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
|
||||
current_app.logger.info(
|
||||
"Job ID {} added to process job queue".format(job.id)
|
||||
)
|
||||
current_app.logger.info("Job ID {} added to process job queue".format(job.id))
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to run scheduled jobs")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name='send-scheduled-notifications')
|
||||
@statsd(namespace="tasks")
|
||||
def send_scheduled_notifications():
|
||||
try:
|
||||
scheduled_notifications = dao_get_scheduled_notifications()
|
||||
for notification in scheduled_notifications:
|
||||
send_notification_to_queue(notification, notification.service.research_mode)
|
||||
set_scheduled_notification_to_processed(notification.id)
|
||||
current_app.logger.info(
|
||||
"Sent {} scheduled notifications to the provider queue".format(len(scheduled_notifications)))
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to send scheduled notifications")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-verify-codes")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_verify_codes():
|
||||
try:
|
||||
start = utc_now()
|
||||
start = datetime.utcnow()
|
||||
deleted = delete_codes_older_created_more_than_a_day_ago()
|
||||
current_app.logger.info(
|
||||
"Delete job started {} finished {} deleted {} verify codes".format(
|
||||
start, utc_now(), deleted
|
||||
)
|
||||
"Delete job started {} finished {} deleted {} verify codes".format(start, datetime.utcnow(), deleted)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete verify codes")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="expire-or-delete-invitations")
|
||||
def expire_or_delete_invitations():
|
||||
@notify_celery.task(name="delete-invitations")
|
||||
@statsd(namespace="tasks")
|
||||
def delete_invitations():
|
||||
try:
|
||||
start = utc_now()
|
||||
expired_invites = expire_invitations_created_more_than_two_days_ago()
|
||||
start = datetime.utcnow()
|
||||
deleted_invites = delete_invitations_created_more_than_two_days_ago()
|
||||
deleted_invites += delete_org_invitations_created_more_than_two_days_ago()
|
||||
current_app.logger.info(
|
||||
f"Expire job started {start} finished {utc_now()} expired {expired_invites} invitations"
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to expire invitations")
|
||||
raise
|
||||
|
||||
try:
|
||||
start = utc_now()
|
||||
deleted_invites = delete_org_invitations_created_more_than_two_days_ago()
|
||||
current_app.logger.info(
|
||||
f"Delete job started {start} finished {utc_now()} deleted {deleted_invites} invitations"
|
||||
"Delete job started {} finished {} deleted {} invitations".format(start, datetime.utcnow(), deleted_invites)
|
||||
)
|
||||
except SQLAlchemyError:
|
||||
current_app.logger.exception("Failed to delete invitations")
|
||||
raise
|
||||
|
||||
|
||||
@notify_celery.task(name="check-job-status")
|
||||
@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
|
||||
@statsd(namespace="tasks")
|
||||
def switch_current_sms_provider_on_slow_delivery():
|
||||
"""
|
||||
Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
|
||||
in the last ten minutes. If both providers are slow, don't do anything. If we changed the providers in the
|
||||
last ten minutes, then don't update them again either.
|
||||
"""
|
||||
slow_delivery_notifications = is_delivery_slow_for_providers(
|
||||
threshold=0.3,
|
||||
created_at=datetime.utcnow() - timedelta(minutes=10),
|
||||
delivery_time=timedelta(minutes=4),
|
||||
)
|
||||
|
||||
# only adjust if some values are true and some are false - ie, don't adjust if all providers are fast or
|
||||
# all providers are slow
|
||||
if len(set(slow_delivery_notifications.values())) != 1:
|
||||
for provider_name, is_slow in slow_delivery_notifications.items():
|
||||
if is_slow:
|
||||
current_app.logger.warning('Slow delivery notifications detected for provider {}'.format(provider_name))
|
||||
dao_reduce_sms_provider_priority(provider_name, time_threshold=timedelta(minutes=10))
|
||||
|
||||
|
||||
@notify_celery.task(name='tend-providers-back-to-middle')
|
||||
@statsd(namespace='tasks')
|
||||
def tend_providers_back_to_middle():
|
||||
dao_adjust_provider_priority_back_to_resting_points()
|
||||
|
||||
|
||||
@notify_celery.task(name='check-job-status')
|
||||
@statsd(namespace="tasks")
|
||||
def check_job_status():
|
||||
"""
|
||||
every x minutes do this check
|
||||
select
|
||||
from jobs
|
||||
where job_status == 'in progress'
|
||||
and processing started some time ago
|
||||
OR where the job_status == 'pending'
|
||||
and the job scheduled_for timestamp is some time ago.
|
||||
and template_type in ('sms', 'email')
|
||||
and scheduled_at or created_at is older that 30 minutes.
|
||||
if any results then
|
||||
update the job_status to 'error'
|
||||
raise error
|
||||
process the rows in the csv that are missing (in another task) just do the check here.
|
||||
"""
|
||||
START_MINUTES = 245
|
||||
END_MINUTES = 240
|
||||
end_minutes_ago = utc_now() - timedelta(minutes=END_MINUTES)
|
||||
start_minutes_ago = utc_now() - timedelta(minutes=START_MINUTES)
|
||||
thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
|
||||
thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
|
||||
|
||||
incomplete_in_progress_jobs = select(Job).where(
|
||||
Job.job_status == JobStatus.IN_PROGRESS,
|
||||
between(Job.processing_started, start_minutes_ago, end_minutes_ago),
|
||||
)
|
||||
incomplete_pending_jobs = select(Job).where(
|
||||
Job.job_status == JobStatus.PENDING,
|
||||
Job.scheduled_for.isnot(None),
|
||||
between(Job.scheduled_for, start_minutes_ago, end_minutes_ago),
|
||||
)
|
||||
jobs_not_completed_after_allotted_time = union(
|
||||
incomplete_in_progress_jobs, incomplete_pending_jobs
|
||||
)
|
||||
jobs_not_completed_after_allotted_time = (
|
||||
jobs_not_completed_after_allotted_time.order_by(
|
||||
Job.processing_started, Job.scheduled_for
|
||||
)
|
||||
)
|
||||
|
||||
jobs_not_complete_after_allotted_time = db.session.execute(
|
||||
jobs_not_completed_after_allotted_time
|
||||
).all()
|
||||
jobs_not_complete_after_30_minutes = Job.query.filter(
|
||||
Job.job_status == JOB_STATUS_IN_PROGRESS,
|
||||
and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
|
||||
).order_by(Job.processing_started).all()
|
||||
|
||||
# temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
|
||||
# if they haven't been re-processed in time.
|
||||
job_ids = []
|
||||
for job in jobs_not_complete_after_allotted_time:
|
||||
dao_update_job_status_to_error(job)
|
||||
for job in jobs_not_complete_after_30_minutes:
|
||||
job.job_status = JOB_STATUS_ERROR
|
||||
dao_update_job(job)
|
||||
job_ids.append(str(job.id))
|
||||
|
||||
if job_ids:
|
||||
current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
|
||||
process_incomplete_jobs.apply_async([job_ids], queue=QueueNames.JOBS)
|
||||
notify_celery.send_task(
|
||||
name=TaskNames.PROCESS_INCOMPLETE_JOBS,
|
||||
args=(job_ids,),
|
||||
queue=QueueNames.JOBS
|
||||
)
|
||||
raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
|
||||
|
||||
|
||||
@notify_celery.task(name="replay-created-notifications")
|
||||
@notify_celery.task(name='replay-created-notifications')
|
||||
@statsd(namespace="tasks")
|
||||
def replay_created_notifications():
|
||||
# if the notification has not be send after 1 hour, then try to resend.
|
||||
resend_created_notifications_older_than = 60 * 60
|
||||
for notification_type in (NotificationType.EMAIL, NotificationType.SMS):
|
||||
resend_created_notifications_older_than = (60 * 60)
|
||||
for notification_type in (EMAIL_TYPE, SMS_TYPE):
|
||||
notifications_to_resend = notifications_not_yet_sent(
|
||||
resend_created_notifications_older_than, notification_type
|
||||
resend_created_notifications_older_than,
|
||||
notification_type
|
||||
)
|
||||
|
||||
if len(notifications_to_resend) > 0:
|
||||
current_app.logger.info(
|
||||
"Sending {} {} notifications "
|
||||
"to the delivery queue because the notification "
|
||||
"status was created.".format(
|
||||
len(notifications_to_resend), notification_type
|
||||
)
|
||||
)
|
||||
current_app.logger.info("Sending {} {} notifications "
|
||||
"to the delivery queue because the notification "
|
||||
"status was created.".format(len(notifications_to_resend), notification_type))
|
||||
|
||||
for n in notifications_to_resend:
|
||||
send_notification_to_queue(notification=n)
|
||||
send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
|
||||
|
||||
# if the letter has not be send after an hour, then create a zendesk ticket
|
||||
letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than)
|
||||
|
||||
if len(letters) > 0:
|
||||
msg = "{} letters were created over an hour ago, " \
|
||||
"but do not have an updated_at timestamp or billable units. " \
|
||||
"\n Creating app.celery.letters_pdf_tasks.create_letters tasks to upload letter to S3 " \
|
||||
"and update notifications for the following notification ids: " \
|
||||
"\n {}".format(len(letters), [x.id for x in letters])
|
||||
|
||||
current_app.logger.info(msg)
|
||||
for letter in letters:
|
||||
create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)
|
||||
|
||||
|
||||
@notify_celery.task(name="check-for-missing-rows-in-completed-jobs")
|
||||
@notify_celery.task(name='check-precompiled-letter-state')
|
||||
@statsd(namespace="tasks")
|
||||
def check_precompiled_letter_state():
|
||||
letters = dao_precompiled_letters_still_pending_virus_check()
|
||||
|
||||
if len(letters) > 0:
|
||||
letter_ids = [str(letter.id) for letter in letters]
|
||||
|
||||
msg = "{} precompiled letters have been pending-virus-check for over 90 minutes. " \
|
||||
"Notifications: {}".format(len(letters), letter_ids)
|
||||
|
||||
current_app.logger.exception(msg)
|
||||
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
|
||||
zendesk_client.create_ticket(
|
||||
subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
|
||||
message=msg,
|
||||
ticket_type=zendesk_client.TYPE_INCIDENT
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name='check-templated-letter-state')
|
||||
@statsd(namespace="tasks")
|
||||
def check_templated_letter_state():
|
||||
letters = dao_old_letters_with_created_status()
|
||||
|
||||
if len(letters) > 0:
|
||||
letter_ids = [str(letter.id) for letter in letters]
|
||||
|
||||
msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
|
||||
"Notifications: {}".format(len(letters), letter_ids)
|
||||
|
||||
current_app.logger.exception(msg)
|
||||
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
|
||||
zendesk_client.create_ticket(
|
||||
subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
|
||||
message=msg,
|
||||
ticket_type=zendesk_client.TYPE_INCIDENT
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
|
||||
def check_for_missing_rows_in_completed_jobs():
|
||||
|
||||
jobs = find_jobs_with_missing_rows()
|
||||
for job in jobs:
|
||||
(
|
||||
recipient_csv,
|
||||
template,
|
||||
sender_id,
|
||||
) = get_recipient_csv_and_template_and_sender_id(job)
|
||||
jobs_and_job_size = find_jobs_with_missing_rows()
|
||||
for x in jobs_and_job_size:
|
||||
job = x[1]
|
||||
missing_rows = find_missing_row_for_job(job.id, job.notification_count)
|
||||
for row_to_process in missing_rows:
|
||||
row = recipient_csv[row_to_process.missing_row]
|
||||
current_app.logger.info(
|
||||
f"Processing missing row: {row_to_process.missing_row} for job: {job.id}"
|
||||
)
|
||||
process_row(row, template, job, job.service, sender_id=sender_id)
|
||||
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
|
||||
for row in recipient_csv.get_rows():
|
||||
if row.index == row_to_process.missing_row:
|
||||
current_app.logger.info(
|
||||
"Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
|
||||
process_row(row, template, job, job.service, sender_id=sender_id)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
name="check-for-services-with-high-failure-rates-or-sending-to-tv-numbers"
|
||||
)
|
||||
@notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
|
||||
@statsd(namespace="tasks")
|
||||
def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
|
||||
start_date = utc_now() - timedelta(days=1)
|
||||
end_date = utc_now()
|
||||
start_date = (datetime.utcnow() - timedelta(days=1))
|
||||
end_date = datetime.utcnow()
|
||||
message = ""
|
||||
|
||||
services_with_failures = dao_find_services_with_high_failure_rates(
|
||||
start_date=start_date, end_date=end_date
|
||||
)
|
||||
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(
|
||||
start_date=start_date, end_date=end_date
|
||||
)
|
||||
services_with_failures = dao_find_services_with_high_failure_rates(start_date=start_date, end_date=end_date)
|
||||
services_sending_to_tv_numbers = dao_find_services_sending_to_tv_numbers(start_date=start_date, end_date=end_date)
|
||||
|
||||
if services_with_failures:
|
||||
message += "{} service(s) have had high permanent-failure rates for sms messages in last 24 hours:\n".format(
|
||||
@@ -210,19 +281,17 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
|
||||
)
|
||||
for service in services_with_failures:
|
||||
service_dashboard = "{}/services/{}".format(
|
||||
current_app.config["ADMIN_BASE_URL"],
|
||||
current_app.config['ADMIN_BASE_URL'],
|
||||
str(service.service_id),
|
||||
)
|
||||
message += "service: {} failure rate: {},\n".format(
|
||||
service_dashboard, service.permanent_failure_rate
|
||||
)
|
||||
message += "service: {} failure rate: {},\n".format(service_dashboard, service.permanent_failure_rate)
|
||||
elif services_sending_to_tv_numbers:
|
||||
message += "{} service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n".format(
|
||||
message += "{} service(s) have sent over 100 sms messages to tv numbers in last 24 hours:\n".format(
|
||||
len(services_sending_to_tv_numbers)
|
||||
)
|
||||
for service in services_sending_to_tv_numbers:
|
||||
service_dashboard = "{}/services/{}".format(
|
||||
current_app.config["ADMIN_BASE_URL"],
|
||||
current_app.config['ADMIN_BASE_URL'],
|
||||
str(service.service_id),
|
||||
)
|
||||
message += "service: {} count of sms to tv numbers: {},\n".format(
|
||||
@@ -232,121 +301,13 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
|
||||
if services_with_failures or services_sending_to_tv_numbers:
|
||||
current_app.logger.warning(message)
|
||||
|
||||
if current_app.config["NOTIFY_ENVIRONMENT"] in ["live", "production", "test"]:
|
||||
message += (
|
||||
"\nYou can find instructions for this ticket in our manual:\n"
|
||||
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers" # noqa
|
||||
)
|
||||
ticket = NotifySupportTicket(
|
||||
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
|
||||
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
|
||||
message += ("\nYou can find instructions for this ticket in our manual:\n"
|
||||
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers") # noqa
|
||||
zendesk_client.create_ticket(
|
||||
subject="[{}] High failure rates for sms spotted for services".format(
|
||||
current_app.config['NOTIFY_ENVIRONMENT']
|
||||
),
|
||||
message=message,
|
||||
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
|
||||
technical_ticket=True,
|
||||
ticket_type=zendesk_client.TYPE_INCIDENT
|
||||
)
|
||||
zendesk_client.send_ticket_to_zendesk(ticket)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, max_retries=7, default_retry_delay=3600, name="process-delivery-receipts"
|
||||
)
|
||||
def process_delivery_receipts(self):
|
||||
# If we need to check db settings do it here for convenience
|
||||
# current_app.logger.info(f"POOL SIZE {app.db.engine.pool.size()}")
|
||||
"""
|
||||
Every eight minutes or so (see config.py) we run this task, which searches the last ten
|
||||
minutes of logs for delivery receipts and batch updates the db with the results. The overlap
|
||||
is intentional. We don't mind re-updating things, it is better than losing data.
|
||||
|
||||
We also set this to retry with exponential backoff in the case of failure. The only way this would
|
||||
fail is if, for example the db went down, or redis filled causing the app to stop processing. But if
|
||||
it does fail, we need to go back over at some point when things are running again and process those results.
|
||||
"""
|
||||
try:
|
||||
batch_size = 1000 # in theory with postgresql this could be 10k to 20k?
|
||||
|
||||
cloudwatch = AwsCloudwatchClient()
|
||||
cloudwatch.init_app(current_app)
|
||||
start_time = aware_utcnow() - timedelta(minutes=3)
|
||||
end_time = aware_utcnow()
|
||||
delivered_receipts, failed_receipts = cloudwatch.check_delivery_receipts(
|
||||
start_time, end_time
|
||||
)
|
||||
delivered_receipts = list(delivered_receipts)
|
||||
for i in range(0, len(delivered_receipts), batch_size):
|
||||
batch = delivered_receipts[i : i + batch_size]
|
||||
dao_update_delivery_receipts(batch, True)
|
||||
failed_receipts = list(failed_receipts)
|
||||
for i in range(0, len(failed_receipts), batch_size):
|
||||
batch = failed_receipts[i : i + batch_size]
|
||||
dao_update_delivery_receipts(batch, False)
|
||||
except Exception as ex:
|
||||
retry_count = self.request.retries
|
||||
wait_time = 3600 * 2**retry_count
|
||||
try:
|
||||
raise self.retry(ex=ex, countdown=wait_time)
|
||||
except self.MaxRetriesExceededError:
|
||||
current_app.logger.error(
|
||||
"Failed process delivery receipts after max retries"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, max_retries=2, default_retry_delay=3600, name="cleanup-delivery-receipts"
|
||||
)
|
||||
def cleanup_delivery_receipts(self):
|
||||
dao_close_out_delivery_receipts()
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="batch-insert-notifications")
|
||||
def batch_insert_notifications(self):
|
||||
batch = []
|
||||
|
||||
# TODO We probably need some way to clear the list if
|
||||
# things go haywire. A command?
|
||||
|
||||
# with redis_store.pipeline():
|
||||
# while redis_store.llen("message_queue") > 0:
|
||||
# redis_store.lpop("message_queue")
|
||||
# current_app.logger.info("EMPTY!")
|
||||
# return
|
||||
current_len = redis_store.llen("message_queue")
|
||||
with redis_store.pipeline():
|
||||
# since this list is being fed by other processes, just grab what is available when
|
||||
# this call is made and process that.
|
||||
|
||||
count = 0
|
||||
while count < current_len:
|
||||
count = count + 1
|
||||
notification_bytes = redis_store.lpop("message_queue")
|
||||
notification_dict = json.loads(notification_bytes.decode("utf-8"))
|
||||
notification_dict["status"] = notification_dict.pop("notification_status")
|
||||
if not notification_dict.get("created_at"):
|
||||
notification_dict["created_at"] = utc_now()
|
||||
elif isinstance(notification_dict["created_at"], list):
|
||||
notification_dict["created_at"] = notification_dict["created_at"][0]
|
||||
notification = Notification(**notification_dict)
|
||||
# notify-api-749 do not write to db
|
||||
# if we have a verify_code we know this is the authentication notification at login time
|
||||
# and not csv (containing PII) provided by the user, so allow verify_code to continue to exist
|
||||
if notification is None:
|
||||
continue
|
||||
if "verify_code" in str(notification.personalisation):
|
||||
pass
|
||||
else:
|
||||
batch.append(notification)
|
||||
try:
|
||||
dao_batch_insert_notifications(batch)
|
||||
except Exception:
|
||||
current_app.logger.exception("Notification batch insert failed")
|
||||
for n in batch:
|
||||
# Use 'created_at' as a TTL so we don't retry infinitely
|
||||
notification_time = n.created_at
|
||||
if isinstance(notification_time, str):
|
||||
notification_time = datetime.fromisoformat(n.created_at)
|
||||
if notification_time < utc_now() - timedelta(seconds=50):
|
||||
current_app.logger.warning(
|
||||
f"Abandoning stale data, could not write to db: {n.serialize_for_redis(n)}"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
redis_store.rpush("message_queue", json.dumps(n.serialize_for_redis(n)))
|
||||
|
||||
@@ -1,147 +1,130 @@
|
||||
import json
|
||||
|
||||
from flask import current_app
|
||||
from requests import HTTPError, RequestException, request
|
||||
|
||||
from app import encryption, notify_celery
|
||||
from app.config import QueueNames
|
||||
from app.utils import DATETIME_FORMAT
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from requests import (
|
||||
HTTPError,
|
||||
request,
|
||||
RequestException
|
||||
)
|
||||
def send_delivery_status_to_service(self, notification_id, encrypted_status_update):
|
||||
|
||||
from app import (
|
||||
notify_celery,
|
||||
encryption
|
||||
)
|
||||
from app.config import QueueNames
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def send_delivery_status_to_service(
|
||||
self, notification_id, encrypted_status_update
|
||||
):
|
||||
status_update = encryption.decrypt(encrypted_status_update)
|
||||
|
||||
data = {
|
||||
"id": str(notification_id),
|
||||
"reference": status_update["notification_client_reference"],
|
||||
"to": status_update["notification_to"],
|
||||
"status": status_update["notification_status"],
|
||||
"created_at": status_update["notification_created_at"],
|
||||
"completed_at": status_update["notification_updated_at"],
|
||||
"sent_at": status_update["notification_sent_at"],
|
||||
"notification_type": status_update["notification_type"],
|
||||
"template_id": status_update["template_id"],
|
||||
"template_version": status_update["template_version"],
|
||||
"reference": status_update['notification_client_reference'],
|
||||
"to": status_update['notification_to'],
|
||||
"status": status_update['notification_status'],
|
||||
"created_at": status_update['notification_created_at'],
|
||||
"completed_at": status_update['notification_updated_at'],
|
||||
"sent_at": status_update['notification_sent_at'],
|
||||
"notification_type": status_update['notification_type']
|
||||
}
|
||||
|
||||
_send_data_to_service_callback_api(
|
||||
self,
|
||||
data,
|
||||
status_update["service_callback_api_url"],
|
||||
status_update["service_callback_api_bearer_token"],
|
||||
"send_delivery_status_to_service",
|
||||
status_update['service_callback_api_url'],
|
||||
status_update['service_callback_api_bearer_token'],
|
||||
'send_delivery_status_to_service'
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="send-complaint", max_retries=5, default_retry_delay=300
|
||||
)
|
||||
@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def send_complaint_to_service(self, complaint_data):
|
||||
complaint = encryption.decrypt(complaint_data)
|
||||
|
||||
data = {
|
||||
"notification_id": complaint["notification_id"],
|
||||
"complaint_id": complaint["complaint_id"],
|
||||
"reference": complaint["reference"],
|
||||
"to": complaint["to"],
|
||||
"complaint_date": complaint["complaint_date"],
|
||||
"notification_id": complaint['notification_id'],
|
||||
"complaint_id": complaint['complaint_id'],
|
||||
"reference": complaint['reference'],
|
||||
"to": complaint['to'],
|
||||
"complaint_date": complaint['complaint_date']
|
||||
}
|
||||
|
||||
_send_data_to_service_callback_api(
|
||||
self,
|
||||
data,
|
||||
complaint["service_callback_api_url"],
|
||||
complaint["service_callback_api_bearer_token"],
|
||||
"send_complaint_to_service",
|
||||
complaint['service_callback_api_url'],
|
||||
complaint['service_callback_api_bearer_token'],
|
||||
'send_complaint_to_service'
|
||||
)
|
||||
|
||||
|
||||
def _send_data_to_service_callback_api(
|
||||
self, data, service_callback_url, token, function_name
|
||||
):
|
||||
notification_id = (
|
||||
data["notification_id"] if "notification_id" in data else data["id"]
|
||||
)
|
||||
def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name):
|
||||
notification_id = (data["notification_id"] if "notification_id" in data else data["id"])
|
||||
try:
|
||||
response = request(
|
||||
method="POST",
|
||||
url=service_callback_url,
|
||||
data=json.dumps(data),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer {}".format(token),
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer {}'.format(token)
|
||||
},
|
||||
timeout=5,
|
||||
)
|
||||
current_app.logger.info(
|
||||
"{} sending {} to {}, response {}".format(
|
||||
function_name,
|
||||
notification_id,
|
||||
service_callback_url,
|
||||
response.status_code,
|
||||
)
|
||||
timeout=60
|
||||
)
|
||||
current_app.logger.info('{} sending {} to {}, response {}'.format(
|
||||
function_name,
|
||||
notification_id,
|
||||
service_callback_url,
|
||||
response.status_code
|
||||
))
|
||||
response.raise_for_status()
|
||||
except RequestException as e:
|
||||
current_app.logger.warning(
|
||||
"{} request failed for notification_id: {} and url: {}. exception: {}".format(
|
||||
function_name, notification_id, service_callback_url, e
|
||||
"{} request failed for notification_id: {} and url: {}. exc: {}".format(
|
||||
function_name,
|
||||
notification_id,
|
||||
service_callback_url,
|
||||
e
|
||||
)
|
||||
)
|
||||
if (
|
||||
not isinstance(e, HTTPError)
|
||||
or e.response.status_code >= 500
|
||||
or e.response.status_code == 429
|
||||
):
|
||||
if not isinstance(e, HTTPError) or e.response.status_code >= 500:
|
||||
try:
|
||||
self.retry(queue=QueueNames.CALLBACKS_RETRY)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
current_app.logger.warning(
|
||||
"Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
|
||||
function_name, service_callback_url, notification_id
|
||||
function_name,
|
||||
service_callback_url,
|
||||
notification_id
|
||||
)
|
||||
)
|
||||
else:
|
||||
current_app.logger.warning(
|
||||
"{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
|
||||
function_name, notification_id, service_callback_url, e
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def create_delivery_status_callback_data(notification, service_callback_api):
|
||||
from app import DATETIME_FORMAT, encryption
|
||||
data = {
|
||||
"notification_id": str(notification.id),
|
||||
"notification_client_reference": notification.client_reference,
|
||||
"notification_to": notification.to,
|
||||
"notification_status": notification.status,
|
||||
"notification_provider_response": notification.provider_response, # TODO do we test for provider_response?
|
||||
"notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
|
||||
"notification_updated_at": (
|
||||
notification.updated_at.strftime(DATETIME_FORMAT)
|
||||
if notification.updated_at
|
||||
else None
|
||||
),
|
||||
"notification_sent_at": (
|
||||
notification.sent_at.strftime(DATETIME_FORMAT)
|
||||
if notification.sent_at
|
||||
else None
|
||||
),
|
||||
"notification_updated_at":
|
||||
notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
|
||||
"notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None,
|
||||
"notification_type": notification.notification_type,
|
||||
"service_callback_api_url": service_callback_api.url,
|
||||
"service_callback_api_bearer_token": service_callback_api.bearer_token,
|
||||
"template_id": str(notification.template_id),
|
||||
"template_version": notification.template_version,
|
||||
}
|
||||
return encryption.encrypt(data)
|
||||
|
||||
|
||||
def create_complaint_callback_data(
|
||||
complaint, notification, service_callback_api, recipient
|
||||
):
|
||||
def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
|
||||
from app import DATETIME_FORMAT, encryption
|
||||
data = {
|
||||
"complaint_id": str(complaint.id),
|
||||
"notification_id": str(notification.id),
|
||||
|
||||
@@ -1,99 +1,117 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from collections import namedtuple, defaultdict
|
||||
|
||||
import eventlet
|
||||
from celery.signals import task_postrun
|
||||
from flask import current_app
|
||||
from requests import HTTPError, RequestException, request
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||
|
||||
from app import create_uuid, encryption, notify_celery
|
||||
from app.aws import s3
|
||||
from app.celery import provider_tasks
|
||||
from app.config import Config, QueueNames
|
||||
from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
|
||||
from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
|
||||
from app.dao.notifications_dao import (
|
||||
dao_get_last_notification_added_for_job_id,
|
||||
get_notification_by_id,
|
||||
from notifications_utils.recipients import (
|
||||
RecipientCSV
|
||||
)
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from notifications_utils.template import (
|
||||
SMSMessageTemplate,
|
||||
WithSubjectTemplate,
|
||||
)
|
||||
from notifications_utils.timezones import convert_utc_to_bst
|
||||
from requests import (
|
||||
HTTPError,
|
||||
request,
|
||||
RequestException
|
||||
)
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from app import (
|
||||
create_uuid,
|
||||
create_random_identifier,
|
||||
DATETIME_FORMAT,
|
||||
encryption,
|
||||
notify_celery,
|
||||
)
|
||||
from app.aws import s3
|
||||
from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
|
||||
from app.config import QueueNames
|
||||
from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
|
||||
from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
|
||||
from app.dao.jobs_dao import (
|
||||
dao_update_job,
|
||||
dao_get_job_by_id,
|
||||
)
|
||||
from app.dao.notifications_dao import (
|
||||
get_notification_by_id,
|
||||
dao_update_notifications_by_reference,
|
||||
dao_get_last_notification_added_for_job_id,
|
||||
update_notification_status_by_reference,
|
||||
dao_get_notification_history_by_reference,
|
||||
)
|
||||
from app.dao.provider_details_dao import get_provider_details_by_notification_type
|
||||
from app.dao.returned_letters_dao import insert_or_update_returned_letters
|
||||
from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
|
||||
from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
|
||||
from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
|
||||
from app.dao.services_dao import dao_fetch_service_by_id
|
||||
from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
|
||||
from app.dao.templates_dao import dao_get_template_by_id
|
||||
from app.enums import JobStatus, KeyType, NotificationType
|
||||
from app.errors import TotalRequestsError
|
||||
from app.notifications.process_notifications import (
|
||||
get_notification,
|
||||
persist_notification,
|
||||
from app.exceptions import DVLAException, NotificationTechnicalFailureException
|
||||
from app.models import (
|
||||
DVLA_RESPONSE_STATUS_SENT,
|
||||
EMAIL_TYPE,
|
||||
JOB_STATUS_CANCELLED,
|
||||
JOB_STATUS_FINISHED,
|
||||
JOB_STATUS_IN_PROGRESS,
|
||||
JOB_STATUS_PENDING,
|
||||
KEY_TYPE_NORMAL,
|
||||
LETTER_TYPE,
|
||||
NOTIFICATION_CREATED,
|
||||
NOTIFICATION_DELIVERED,
|
||||
NOTIFICATION_SENDING,
|
||||
NOTIFICATION_TEMPORARY_FAILURE,
|
||||
NOTIFICATION_TECHNICAL_FAILURE,
|
||||
NOTIFICATION_RETURNED_LETTER,
|
||||
SMS_TYPE,
|
||||
DailySortedLetter,
|
||||
)
|
||||
from app.notifications.validators import check_service_over_total_message_limit
|
||||
from app.serialised_models import SerialisedService, SerialisedTemplate
|
||||
from app.notifications.process_notifications import persist_notification
|
||||
from app.service.utils import service_allowed_to_send_to
|
||||
from app.utils import DATETIME_FORMAT, hilite, utc_now
|
||||
from notifications_utils.recipients import RecipientCSV
|
||||
|
||||
|
||||
@notify_celery.task(name="process-job")
|
||||
@statsd(namespace="tasks")
|
||||
def process_job(job_id, sender_id=None):
|
||||
"""Update job status, get csv data from s3, and begin processing csv rows."""
|
||||
start = utc_now()
|
||||
start = datetime.utcnow()
|
||||
job = dao_get_job_by_id(job_id)
|
||||
current_app.logger.info(
|
||||
f"Starting process-job task for job id {job_id} with status: {job.job_status}"
|
||||
)
|
||||
current_app.logger.info("Starting process-job task for job id {} with status: {}".format(job_id, job.job_status))
|
||||
|
||||
if job.job_status != JobStatus.PENDING:
|
||||
if job.job_status != JOB_STATUS_PENDING:
|
||||
return
|
||||
|
||||
service = job.service
|
||||
|
||||
job.job_status = JobStatus.IN_PROGRESS
|
||||
if not service.active:
|
||||
job.job_status = JOB_STATUS_CANCELLED
|
||||
dao_update_job(job)
|
||||
current_app.logger.warning(
|
||||
"Job {} has been cancelled, service {} is inactive".format(job_id, service.id))
|
||||
return
|
||||
|
||||
if __sending_limits_for_job_exceeded(service, job, job_id):
|
||||
return
|
||||
|
||||
job.job_status = JOB_STATUS_IN_PROGRESS
|
||||
job.processing_started = start
|
||||
dao_update_job(job)
|
||||
|
||||
if not service.active:
|
||||
job.job_status = JobStatus.CANCELLED
|
||||
dao_update_job(job)
|
||||
current_app.logger.warning(
|
||||
f"Job {job_id} has been cancelled, service {service.id} is inactive".format(
|
||||
job_id, service.id
|
||||
)
|
||||
)
|
||||
return
|
||||
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
|
||||
|
||||
if __total_sending_limits_for_job_exceeded(service, job, job_id):
|
||||
return
|
||||
current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
|
||||
|
||||
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(
|
||||
job
|
||||
)
|
||||
|
||||
current_app.logger.info(
|
||||
f"Starting job {job_id} processing {job.notification_count} notifications"
|
||||
)
|
||||
|
||||
# notify-api-1495 we are going to sleep periodically to give other
|
||||
# jobs running at the same time a chance to get some of their messages
|
||||
# sent. Sleep for 1 second after every 3 sends, which gives us throughput
|
||||
# of about 3600*3 per hour and would keep the queue clear assuming only one sender.
|
||||
# It will also hopefully eliminate throttling when we send messages which we are
|
||||
# currently seeing.
|
||||
count = 0
|
||||
for row in recipient_csv.get_rows():
|
||||
process_row(row, template, job, service, sender_id=sender_id)
|
||||
count = count + 1
|
||||
if count % 3 == 0:
|
||||
eventlet.sleep(1)
|
||||
|
||||
# End point/Exit point for message send flow.
|
||||
job_complete(job, start=start)
|
||||
|
||||
|
||||
def job_complete(job, resumed=False, start=None):
|
||||
job.job_status = JobStatus.FINISHED
|
||||
job.job_status = JOB_STATUS_FINISHED
|
||||
|
||||
finished = utc_now()
|
||||
finished = datetime.utcnow()
|
||||
job.processing_finished = finished
|
||||
dao_update_job(job)
|
||||
|
||||
@@ -103,50 +121,47 @@ def job_complete(job, resumed=False, start=None):
|
||||
)
|
||||
else:
|
||||
current_app.logger.info(
|
||||
"Job {} created at {} started at {} finished at {}".format(
|
||||
job.id, job.created_at, start, finished
|
||||
)
|
||||
"Job {} created at {} started at {} finished at {}".format(job.id, job.created_at, start, finished)
|
||||
)
|
||||
|
||||
|
||||
def get_recipient_csv_and_template_and_sender_id(job):
|
||||
db_template = dao_get_template_by_id(job.template_id, job.template_version)
|
||||
template = db_template._as_utils_template()
|
||||
|
||||
contents, meta_data = s3.get_job_and_metadata_from_s3(
|
||||
service_id=str(job.service_id), job_id=str(job.id)
|
||||
)
|
||||
recipient_csv = RecipientCSV(contents, template=template)
|
||||
TemplateClass = get_template_class(db_template.template_type)
|
||||
template = TemplateClass(db_template.__dict__)
|
||||
contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id))
|
||||
recipient_csv = RecipientCSV(file_data=contents,
|
||||
template_type=template.template_type,
|
||||
placeholders=template.placeholders)
|
||||
|
||||
return recipient_csv, template, meta_data.get("sender_id")
|
||||
|
||||
|
||||
def process_row(row, template, job, service, sender_id=None):
|
||||
"""Branch off based on notification type, sms or email."""
|
||||
template_type = template.template_type
|
||||
encrypted = encryption.encrypt(
|
||||
{
|
||||
"template": str(template.id),
|
||||
"template_version": job.template_version,
|
||||
"job": str(job.id),
|
||||
"to": row.recipient,
|
||||
"row_number": row.index,
|
||||
"personalisation": dict(row.personalisation),
|
||||
}
|
||||
)
|
||||
encrypted = encryption.encrypt({
|
||||
'template': str(template.id),
|
||||
'template_version': job.template_version,
|
||||
'job': str(job.id),
|
||||
'to': row.recipient,
|
||||
'row_number': row.index,
|
||||
'personalisation': dict(row.personalisation)
|
||||
})
|
||||
|
||||
# Both save_sms and save_email have the same general
|
||||
# persist logic.
|
||||
send_fns = {NotificationType.SMS: save_sms, NotificationType.EMAIL: save_email}
|
||||
send_fns = {
|
||||
SMS_TYPE: save_sms,
|
||||
EMAIL_TYPE: save_email,
|
||||
LETTER_TYPE: save_letter
|
||||
}
|
||||
|
||||
send_fn = send_fns[template_type]
|
||||
|
||||
task_kwargs = {}
|
||||
if sender_id:
|
||||
task_kwargs["sender_id"] = sender_id
|
||||
task_kwargs['sender_id'] = sender_id
|
||||
|
||||
notification_id = create_uuid()
|
||||
# Kick-off persisting notification in save_sms/save_email.
|
||||
send_fn.apply_async(
|
||||
(
|
||||
str(service.id),
|
||||
@@ -154,297 +169,371 @@ def process_row(row, template, job, service, sender_id=None):
|
||||
encrypted,
|
||||
),
|
||||
task_kwargs,
|
||||
queue=QueueNames.DATABASE,
|
||||
expires=Config.DEFAULT_REDIS_EXPIRE_TIME,
|
||||
queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
|
||||
)
|
||||
return notification_id
|
||||
|
||||
|
||||
# TODO
|
||||
# Originally this was checking a daily limit
|
||||
# It is now checking an overall limit (annual?) for the free tier
|
||||
# Is there any limit for the paid tier?
|
||||
# Assuming the limit is annual, is it calendar year, fiscal year, MOU year?
|
||||
# Do we need a command to run to clear the redis value, or should it happen automatically?
|
||||
def __total_sending_limits_for_job_exceeded(service, job, job_id):
|
||||
def __sending_limits_for_job_exceeded(service, job, job_id):
|
||||
total_sent = fetch_todays_total_message_count(service.id)
|
||||
|
||||
try:
|
||||
total_sent = check_service_over_total_message_limit(KeyType.NORMAL, service)
|
||||
if total_sent + job.notification_count > service.total_message_limit:
|
||||
raise TotalRequestsError(service.total_message_limit)
|
||||
else:
|
||||
return False
|
||||
except TotalRequestsError:
|
||||
job.job_status = "sending limits exceeded"
|
||||
job.processing_finished = utc_now()
|
||||
if total_sent + job.notification_count > service.message_limit:
|
||||
job.job_status = 'sending limits exceeded'
|
||||
job.processing_finished = datetime.utcnow()
|
||||
dao_update_job(job)
|
||||
current_app.logger.exception(
|
||||
"Job {} size {} error. Total sending limits {} exceeded".format(
|
||||
job_id, job.notification_count, service.total_message_limit
|
||||
),
|
||||
current_app.logger.info(
|
||||
"Job {} size {} error. Sending limits {} exceeded".format(
|
||||
job_id, job.notification_count, service.message_limit)
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@task_postrun.connect
|
||||
def log_task_ejection(sender=None, task_id=None, **kwargs):
|
||||
current_app.logger.info(
|
||||
f"Task {task_id} ({sender.name if sender else 'unknown_task'}) has been completed and removed"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="save-sms", max_retries=2, default_retry_delay=600)
|
||||
def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None):
|
||||
"""Persist notification to db and place notification in queue to send to sns."""
|
||||
@notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def save_sms(self,
|
||||
service_id,
|
||||
notification_id,
|
||||
encrypted_notification,
|
||||
sender_id=None):
|
||||
notification = encryption.decrypt(encrypted_notification)
|
||||
# SerialisedService and SerialisedTemplate classes are
|
||||
# used here to grab the same service and template from the cache
|
||||
# to improve performance.
|
||||
service = SerialisedService.from_id(service_id)
|
||||
template = SerialisedTemplate.from_id_and_service_id(
|
||||
notification["template"],
|
||||
service_id=service.id,
|
||||
version=notification["template_version"],
|
||||
)
|
||||
service = dao_fetch_service_by_id(service_id)
|
||||
template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
|
||||
|
||||
if sender_id:
|
||||
reply_to_text = dao_get_service_sms_senders_by_id(
|
||||
service_id, sender_id
|
||||
).sms_sender
|
||||
reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
|
||||
else:
|
||||
reply_to_text = template.reply_to_text
|
||||
# Return False when trial mode services try sending notifications
|
||||
# to non-team and non-simulated recipients.
|
||||
if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
|
||||
current_app.logger.info(
|
||||
hilite(
|
||||
f"service not allowed to send for job_id {notification.get('job', None)}, aborting"
|
||||
)
|
||||
reply_to_text = template.get_reply_to_text()
|
||||
|
||||
if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
|
||||
current_app.logger.debug(
|
||||
"SMS {} failed as restricted service".format(notification_id)
|
||||
)
|
||||
current_app.logger.debug(f"SMS {notification_id} failed as restricted service")
|
||||
return
|
||||
|
||||
try:
|
||||
job_id = notification.get("job", None)
|
||||
created_by_id = None
|
||||
if job_id:
|
||||
job = dao_get_job_by_id(job_id)
|
||||
created_by_id = job.created_by_id
|
||||
|
||||
try:
|
||||
saved_notification = persist_notification(
|
||||
template_id=notification["template"],
|
||||
template_version=notification["template_version"],
|
||||
recipient=notification["to"],
|
||||
service=service,
|
||||
personalisation=notification.get("personalisation"),
|
||||
notification_type=NotificationType.SMS,
|
||||
api_key_id=None,
|
||||
key_type=KeyType.NORMAL,
|
||||
created_at=utc_now(),
|
||||
created_by_id=created_by_id,
|
||||
job_id=notification.get("job", None),
|
||||
job_row_number=notification.get("row_number", None),
|
||||
notification_id=notification_id,
|
||||
reply_to_text=reply_to_text,
|
||||
)
|
||||
except IntegrityError:
|
||||
current_app.logger.warning(
|
||||
f"{NotificationType.SMS}: {notification_id} already exists."
|
||||
)
|
||||
# If we don't have the return statement here, we will fall through and end
|
||||
# up retrying because IntegrityError is a subclass of SQLAlchemyError
|
||||
return
|
||||
|
||||
# Kick off sns process in provider_tasks.py
|
||||
sn = saved_notification
|
||||
current_app.logger.info(
|
||||
hilite(
|
||||
f"Deliver sms for job_id: {sn.job_id} row_number: {sn.job_row_number}"
|
||||
)
|
||||
saved_notification = persist_notification(
|
||||
template_id=notification['template'],
|
||||
template_version=notification['template_version'],
|
||||
recipient=notification['to'],
|
||||
service=service,
|
||||
personalisation=notification.get('personalisation'),
|
||||
notification_type=SMS_TYPE,
|
||||
api_key_id=None,
|
||||
key_type=KEY_TYPE_NORMAL,
|
||||
created_at=datetime.utcnow(),
|
||||
job_id=notification.get('job', None),
|
||||
job_row_number=notification.get('row_number', None),
|
||||
notification_id=notification_id,
|
||||
reply_to_text=reply_to_text
|
||||
)
|
||||
|
||||
provider_tasks.deliver_sms.apply_async(
|
||||
[str(saved_notification.id)], queue=QueueNames.SEND_SMS, countdown=60
|
||||
[str(saved_notification.id)],
|
||||
queue=QueueNames.SEND_SMS if not service.research_mode else QueueNames.RESEARCH_MODE
|
||||
)
|
||||
|
||||
current_app.logger.debug(
|
||||
f"SMS {saved_notification.id} created at {saved_notification.created_at} "
|
||||
f"for job {notification.get('job', None)}"
|
||||
"SMS {} created at {} for job {}".format(
|
||||
saved_notification.id,
|
||||
saved_notification.created_at,
|
||||
notification.get('job', None))
|
||||
)
|
||||
|
||||
except SQLAlchemyError as e:
|
||||
handle_exception(self, notification, notification_id, e)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="save-email", max_retries=5, default_retry_delay=300
|
||||
)
|
||||
def save_email(
|
||||
self, service_id, notification_id, encrypted_notification, sender_id=None
|
||||
):
|
||||
@notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def save_email(self,
|
||||
service_id,
|
||||
notification_id,
|
||||
encrypted_notification,
|
||||
sender_id=None):
|
||||
notification = encryption.decrypt(encrypted_notification)
|
||||
|
||||
service = SerialisedService.from_id(service_id)
|
||||
template = SerialisedTemplate.from_id_and_service_id(
|
||||
notification["template"],
|
||||
service_id=service.id,
|
||||
version=notification["template_version"],
|
||||
)
|
||||
service = dao_fetch_service_by_id(service_id)
|
||||
template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
|
||||
|
||||
if sender_id:
|
||||
reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
|
||||
else:
|
||||
reply_to_text = template.reply_to_text
|
||||
reply_to_text = template.get_reply_to_text()
|
||||
|
||||
if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
|
||||
current_app.logger.info(
|
||||
"Email {} failed as restricted service".format(notification_id)
|
||||
)
|
||||
if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
|
||||
current_app.logger.info("Email {} failed as restricted service".format(notification_id))
|
||||
return
|
||||
original_notification = get_notification(notification_id)
|
||||
|
||||
try:
|
||||
saved_notification = persist_notification(
|
||||
template_id=notification["template"],
|
||||
template_version=notification["template_version"],
|
||||
recipient=notification["to"],
|
||||
template_id=notification['template'],
|
||||
template_version=notification['template_version'],
|
||||
recipient=notification['to'],
|
||||
service=service,
|
||||
personalisation=notification.get("personalisation"),
|
||||
notification_type=NotificationType.EMAIL,
|
||||
personalisation=notification.get('personalisation'),
|
||||
notification_type=EMAIL_TYPE,
|
||||
api_key_id=None,
|
||||
key_type=KeyType.NORMAL,
|
||||
created_at=utc_now(),
|
||||
job_id=notification.get("job", None),
|
||||
job_row_number=notification.get("row_number", None),
|
||||
key_type=KEY_TYPE_NORMAL,
|
||||
created_at=datetime.utcnow(),
|
||||
job_id=notification.get('job', None),
|
||||
job_row_number=notification.get('row_number', None),
|
||||
notification_id=notification_id,
|
||||
reply_to_text=reply_to_text,
|
||||
reply_to_text=reply_to_text
|
||||
)
|
||||
# we only want to send once
|
||||
if original_notification is None:
|
||||
provider_tasks.deliver_email.apply_async(
|
||||
[str(saved_notification.id)], queue=QueueNames.SEND_EMAIL
|
||||
)
|
||||
|
||||
current_app.logger.debug(
|
||||
"Email {} created at {}".format(
|
||||
saved_notification.id, saved_notification.created_at
|
||||
)
|
||||
provider_tasks.deliver_email.apply_async(
|
||||
[str(saved_notification.id)],
|
||||
queue=QueueNames.SEND_EMAIL if not service.research_mode else QueueNames.RESEARCH_MODE
|
||||
)
|
||||
|
||||
current_app.logger.debug("Email {} created at {}".format(saved_notification.id, saved_notification.created_at))
|
||||
except SQLAlchemyError as e:
|
||||
handle_exception(self, notification, notification_id, e)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="save-api-email", max_retries=5, default_retry_delay=300
|
||||
)
|
||||
def save_api_email(self, encrypted_notification):
|
||||
save_api_email_or_sms(self, encrypted_notification)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="save-api-sms", max_retries=2, default_retry_delay=600
|
||||
)
|
||||
def save_api_sms(self, encrypted_notification):
|
||||
save_api_email_or_sms(self, encrypted_notification)
|
||||
|
||||
|
||||
def save_api_email_or_sms(self, encrypted_notification):
|
||||
@notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def save_letter(
|
||||
self,
|
||||
service_id,
|
||||
notification_id,
|
||||
encrypted_notification,
|
||||
):
|
||||
notification = encryption.decrypt(encrypted_notification)
|
||||
service = SerialisedService.from_id(notification["service_id"])
|
||||
q = (
|
||||
QueueNames.SEND_EMAIL
|
||||
if notification["notification_type"] == NotificationType.EMAIL
|
||||
else QueueNames.SEND_SMS
|
||||
)
|
||||
provider_task = (
|
||||
provider_tasks.deliver_email
|
||||
if notification["notification_type"] == NotificationType.EMAIL
|
||||
else provider_tasks.deliver_sms
|
||||
)
|
||||
|
||||
original_notification = get_notification(notification["id"])
|
||||
# we store the recipient as just the first item of the person's address
|
||||
recipient = notification['personalisation']['addressline1']
|
||||
|
||||
service = dao_fetch_service_by_id(service_id)
|
||||
template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
|
||||
|
||||
try:
|
||||
persist_notification(
|
||||
notification_id=notification["id"],
|
||||
template_id=notification["template_id"],
|
||||
template_version=notification["template_version"],
|
||||
recipient=notification["to"],
|
||||
# if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
|
||||
status = NOTIFICATION_CREATED if not service.research_mode else NOTIFICATION_SENDING
|
||||
|
||||
saved_notification = persist_notification(
|
||||
template_id=notification['template'],
|
||||
template_version=notification['template_version'],
|
||||
template_postage=template.postage,
|
||||
recipient=recipient,
|
||||
service=service,
|
||||
personalisation=notification.get("personalisation"),
|
||||
notification_type=notification["notification_type"],
|
||||
client_reference=notification["client_reference"],
|
||||
api_key_id=notification.get("api_key_id"),
|
||||
key_type=KeyType.NORMAL,
|
||||
created_at=notification["created_at"],
|
||||
reply_to_text=notification["reply_to_text"],
|
||||
status=notification["status"],
|
||||
document_download_count=notification["document_download_count"],
|
||||
personalisation=notification['personalisation'],
|
||||
notification_type=LETTER_TYPE,
|
||||
api_key_id=None,
|
||||
key_type=KEY_TYPE_NORMAL,
|
||||
created_at=datetime.utcnow(),
|
||||
job_id=notification['job'],
|
||||
job_row_number=notification['row_number'],
|
||||
notification_id=notification_id,
|
||||
reference=create_random_identifier(),
|
||||
reply_to_text=template.get_reply_to_text(),
|
||||
status=status
|
||||
)
|
||||
# Only get here if save to the db was successful (i.e. first time)
|
||||
if original_notification is None:
|
||||
provider_task.apply_async([notification["id"]], queue=q)
|
||||
current_app.logger.debug(
|
||||
f"{notification['id']} has been persisted and sent to delivery queue."
|
||||
)
|
||||
|
||||
except IntegrityError:
|
||||
current_app.logger.warning(
|
||||
f"{notification['notification_type']} {notification['id']} already exists."
|
||||
)
|
||||
# If we don't have the return statement here, we will fall through and end
|
||||
# up retrying because IntegrityError is a subclass of SQLAlchemyError
|
||||
return
|
||||
|
||||
except SQLAlchemyError:
|
||||
try:
|
||||
self.retry(queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME)
|
||||
except self.MaxRetriesExceededError:
|
||||
current_app.logger.exception(
|
||||
f"Max retry failed Failed to persist notification {notification['id']}",
|
||||
if not service.research_mode:
|
||||
letters_pdf_tasks.create_letters_pdf.apply_async(
|
||||
[str(saved_notification.id)],
|
||||
queue=QueueNames.CREATE_LETTERS_PDF
|
||||
)
|
||||
elif current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']:
|
||||
research_mode_tasks.create_fake_letter_response_file.apply_async(
|
||||
(saved_notification.reference,),
|
||||
queue=QueueNames.RESEARCH_MODE
|
||||
)
|
||||
else:
|
||||
update_notification_status_by_reference(saved_notification.reference, 'delivered')
|
||||
|
||||
current_app.logger.debug("Letter {} created at {}".format(saved_notification.id, saved_notification.created_at))
|
||||
except SQLAlchemyError as e:
|
||||
handle_exception(self, notification, notification_id, e)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
|
||||
@statsd(namespace="tasks")
|
||||
def update_letter_notifications_to_sent_to_dvla(self, notification_references):
|
||||
# This task will be called by the FTP app to update notifications as sent to DVLA
|
||||
provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
|
||||
|
||||
updated_count, _ = dao_update_notifications_by_reference(
|
||||
notification_references,
|
||||
{
|
||||
'status': NOTIFICATION_SENDING,
|
||||
'sent_by': provider.identifier,
|
||||
'sent_at': datetime.utcnow(),
|
||||
'updated_at': datetime.utcnow()
|
||||
}
|
||||
)
|
||||
|
||||
current_app.logger.info("Updated {} letter notifications to sending".format(updated_count))
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='update-letter-notifications-to-error')
|
||||
@statsd(namespace="tasks")
|
||||
def update_letter_notifications_to_error(self, notification_references):
|
||||
# This task will be called by the FTP app to update notifications as sent to DVLA
|
||||
|
||||
updated_count, _ = dao_update_notifications_by_reference(
|
||||
notification_references,
|
||||
{
|
||||
'status': NOTIFICATION_TECHNICAL_FAILURE,
|
||||
'updated_at': datetime.utcnow()
|
||||
}
|
||||
)
|
||||
message = "Updated {} letter notifications to technical-failure with references {}".format(
|
||||
updated_count, notification_references
|
||||
)
|
||||
raise NotificationTechnicalFailureException(message)
|
||||
|
||||
|
||||
def handle_exception(task, notification, notification_id, exc):
|
||||
if not get_notification_by_id(notification_id):
|
||||
retry_msg = "{task} notification for job {job} row number {row} and notification id {noti}".format(
|
||||
retry_msg = '{task} notification for job {job} row number {row} and notification id {noti}'.format(
|
||||
task=task.__name__,
|
||||
job=notification.get("job", None),
|
||||
row=notification.get("row_number", None),
|
||||
noti=notification_id,
|
||||
job=notification.get('job', None),
|
||||
row=notification.get('row_number', None),
|
||||
noti=notification_id
|
||||
)
|
||||
# Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
|
||||
# SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
|
||||
# send to the retry queue.
|
||||
# This probably (hopefully) is not an issue with Redis as the celery backing store
|
||||
current_app.logger.exception("Retry" + retry_msg)
|
||||
current_app.logger.exception('Retry' + retry_msg)
|
||||
try:
|
||||
task.retry(
|
||||
queue=QueueNames.RETRY,
|
||||
exc=exc,
|
||||
expires=Config.DEFAULT_REDIS_EXPIRE_TIME,
|
||||
)
|
||||
task.retry(queue=QueueNames.RETRY, exc=exc)
|
||||
except task.MaxRetriesExceededError:
|
||||
current_app.logger.exception("Max retry failed" + retry_msg)
|
||||
current_app.logger.error('Max retry failed' + retry_msg)
|
||||
|
||||
|
||||
@notify_celery.task(
|
||||
bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300
|
||||
)
|
||||
def get_template_class(template_type):
|
||||
if template_type == SMS_TYPE:
|
||||
return SMSMessageTemplate
|
||||
elif template_type in (EMAIL_TYPE, LETTER_TYPE):
|
||||
# since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
|
||||
# use the same base template
|
||||
return WithSubjectTemplate
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
|
||||
@statsd(namespace="tasks")
|
||||
def update_letter_notifications_statuses(self, filename):
|
||||
notification_updates = parse_dvla_file(filename)
|
||||
|
||||
temporary_failures = []
|
||||
|
||||
for update in notification_updates:
|
||||
check_billable_units(update)
|
||||
update_letter_notification(filename, temporary_failures, update)
|
||||
if temporary_failures:
|
||||
# This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
|
||||
message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \
|
||||
.format(filename=filename, failures=temporary_failures)
|
||||
raise DVLAException(message)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="record-daily-sorted-counts")
|
||||
@statsd(namespace="tasks")
|
||||
def record_daily_sorted_counts(self, filename):
|
||||
sorted_letter_counts = defaultdict(int)
|
||||
notification_updates = parse_dvla_file(filename)
|
||||
for update in notification_updates:
|
||||
sorted_letter_counts[update.cost_threshold.lower()] += 1
|
||||
|
||||
unknown_status = sorted_letter_counts.keys() - {'unsorted', 'sorted'}
|
||||
if unknown_status:
|
||||
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
|
||||
filename, unknown_status.__repr__()
|
||||
)
|
||||
raise DVLAException(message)
|
||||
|
||||
billing_date = get_billing_date_in_bst_from_filename(filename)
|
||||
persist_daily_sorted_letter_counts(day=billing_date,
|
||||
file_name=filename,
|
||||
sorted_letter_counts=sorted_letter_counts)
|
||||
|
||||
|
||||
def parse_dvla_file(filename):
|
||||
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
|
||||
response_file_content = s3.get_s3_file(bucket_location, filename)
|
||||
|
||||
try:
|
||||
return process_updates_from_file(response_file_content)
|
||||
except TypeError:
|
||||
raise DVLAException('DVLA response file: {} has an invalid format'.format(filename))
|
||||
|
||||
|
||||
def get_billing_date_in_bst_from_filename(filename):
|
||||
# exclude seconds from the date since we don't need it. We got a date ending in 60 second - which is not valid.
|
||||
datetime_string = filename.split('-')[1][:-2]
|
||||
datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M')
|
||||
return convert_utc_to_bst(datetime_obj).date()
|
||||
|
||||
|
||||
def persist_daily_sorted_letter_counts(day, file_name, sorted_letter_counts):
|
||||
daily_letter_count = DailySortedLetter(
|
||||
billing_day=day,
|
||||
file_name=file_name,
|
||||
unsorted_count=sorted_letter_counts['unsorted'],
|
||||
sorted_count=sorted_letter_counts['sorted']
|
||||
)
|
||||
dao_create_or_update_daily_sorted_letter(daily_letter_count)
|
||||
|
||||
|
||||
def process_updates_from_file(response_file):
|
||||
NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
|
||||
notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
|
||||
return notification_updates
|
||||
|
||||
|
||||
def update_letter_notification(filename, temporary_failures, update):
|
||||
if update.status == DVLA_RESPONSE_STATUS_SENT:
|
||||
status = NOTIFICATION_DELIVERED
|
||||
else:
|
||||
status = NOTIFICATION_TEMPORARY_FAILURE
|
||||
temporary_failures.append(update.reference)
|
||||
|
||||
updated_count, _ = dao_update_notifications_by_reference(
|
||||
references=[update.reference],
|
||||
update_dict={"status": status,
|
||||
"updated_at": datetime.utcnow()
|
||||
}
|
||||
)
|
||||
|
||||
if not updated_count:
|
||||
msg = "Update letter notification file {filename} failed: notification either not found " \
|
||||
"or already updated from delivered. Status {status} for notification reference {reference}".format(
|
||||
filename=filename, status=status, reference=update.reference)
|
||||
current_app.logger.info(msg)
|
||||
|
||||
|
||||
def check_billable_units(notification_update):
|
||||
notification = dao_get_notification_history_by_reference(notification_update.reference)
|
||||
|
||||
if int(notification_update.page_count) != notification.billable_units:
|
||||
msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(
|
||||
notification.id, notification.billable_units, notification_update.page_count)
|
||||
try:
|
||||
raise DVLAException(msg)
|
||||
except DVLAException:
|
||||
current_app.logger.exception(msg)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
|
||||
inbound_api = get_service_inbound_api_for_service(service_id=service_id)
|
||||
if not inbound_api:
|
||||
# No API data has been set for this service
|
||||
return
|
||||
|
||||
inbound_sms = dao_get_inbound_sms_by_id(
|
||||
service_id=service_id, inbound_id=inbound_sms_id
|
||||
)
|
||||
inbound_sms = dao_get_inbound_sms_by_id(service_id=service_id,
|
||||
inbound_id=inbound_sms_id)
|
||||
data = {
|
||||
"id": str(inbound_sms.id),
|
||||
# TODO: should we be validating and formatting the phone number here?
|
||||
"source_number": inbound_sms.user_number,
|
||||
"destination_number": inbound_sms.notify_number,
|
||||
"message": inbound_sms.content,
|
||||
"date_received": inbound_sms.provider_date.strftime(DATETIME_FORMAT),
|
||||
"date_received": inbound_sms.provider_date.strftime(DATETIME_FORMAT)
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -453,69 +542,48 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
|
||||
url=inbound_api.url,
|
||||
data=json.dumps(data),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer {}".format(inbound_api.bearer_token),
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer {}'.format(inbound_api.bearer_token)
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
current_app.logger.debug(
|
||||
f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, "
|
||||
+ f"response {response.status_code}"
|
||||
timeout=60
|
||||
)
|
||||
current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
|
||||
inbound_sms_id,
|
||||
inbound_api.url,
|
||||
response.status_code
|
||||
))
|
||||
response.raise_for_status()
|
||||
except RequestException as e:
|
||||
current_app.logger.warning(
|
||||
f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} "
|
||||
+ f"and url: {inbound_api.url}. exception: {e}"
|
||||
"send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
|
||||
service_id,
|
||||
inbound_sms_id,
|
||||
inbound_api.url,
|
||||
e
|
||||
)
|
||||
)
|
||||
if not isinstance(e, HTTPError) or e.response.status_code >= 500:
|
||||
try:
|
||||
self.retry(
|
||||
queue=QueueNames.RETRY, expires=Config.DEFAULT_REDIS_EXPIRE_TIME
|
||||
)
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
current_app.logger.exception(
|
||||
"Retry: send_inbound_sms_to_service has retried the max number of"
|
||||
+ f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
|
||||
current_app.logger.error(
|
||||
"""Retry: send_inbound_sms_to_service has retried the max number of
|
||||
times for service: {} and inbound_sms {}""".format(
|
||||
service_id,
|
||||
inbound_sms_id
|
||||
)
|
||||
)
|
||||
else:
|
||||
current_app.logger.warning(
|
||||
f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for "
|
||||
+ f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="regenerate-job-cache")
|
||||
def regenerate_job_cache():
|
||||
s3.get_s3_files()
|
||||
|
||||
|
||||
@notify_celery.task(name="clean-job-cache")
|
||||
def clean_job_cache():
|
||||
s3.clean_cache()
|
||||
|
||||
|
||||
@notify_celery.task(name="delete-old-s3-objects")
|
||||
def delete_old_s3_objects():
|
||||
|
||||
existing_service_ids = s3.cleanup_old_s3_objects()
|
||||
service_names = []
|
||||
for service_id in existing_service_ids:
|
||||
service = dao_fetch_service_by_id(service_id)
|
||||
service_names.append(service.name)
|
||||
current_app.logger.info(
|
||||
f"#delete-old-s3-objects Services with retained csvs: {service_names}"
|
||||
)
|
||||
|
||||
|
||||
@notify_celery.task(name="process-incomplete-jobs")
|
||||
@notify_celery.task(name='process-incomplete-jobs')
|
||||
@statsd(namespace="tasks")
|
||||
def process_incomplete_jobs(job_ids):
|
||||
jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
|
||||
|
||||
# reset the processing start time so that the check_job_status scheduled task doesn't pick this job up again
|
||||
for job in jobs:
|
||||
job.job_status = JobStatus.IN_PROGRESS
|
||||
job.processing_started = utc_now()
|
||||
job.job_status = JOB_STATUS_IN_PROGRESS
|
||||
job.processing_started = datetime.utcnow()
|
||||
dao_update_job(job)
|
||||
|
||||
current_app.logger.info("Resuming Job(s) {}".format(job_ids))
|
||||
@@ -533,16 +601,29 @@ def process_incomplete_job(job_id):
|
||||
else:
|
||||
resume_from_row = -1 # The first row in the csv with a number is row 0
|
||||
|
||||
current_app.logger.info(
|
||||
"Resuming job {} from row {}".format(job_id, resume_from_row)
|
||||
)
|
||||
current_app.logger.info("Resuming job {} from row {}".format(job_id, resume_from_row))
|
||||
|
||||
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(
|
||||
job
|
||||
)
|
||||
recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
|
||||
|
||||
for row in recipient_csv.get_rows():
|
||||
if row.index > resume_from_row:
|
||||
process_row(row, template, job, job.service, sender_id=sender_id)
|
||||
|
||||
job_complete(job, resumed=True)
|
||||
|
||||
|
||||
@notify_celery.task(name='process-returned-letters-list')
|
||||
@statsd(namespace="tasks")
|
||||
def process_returned_letters_list(notification_references):
|
||||
updated, updated_history = dao_update_notifications_by_reference(
|
||||
notification_references,
|
||||
{"status": NOTIFICATION_RETURNED_LETTER}
|
||||
)
|
||||
|
||||
insert_or_update_returned_letters(notification_references)
|
||||
|
||||
current_app.logger.info(
|
||||
"Updated {} letter notifications ({} history notifications, from {} references) to returned-letter".format(
|
||||
updated, updated_history, len(notification_references)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
import json
|
||||
|
||||
from flask import current_app
|
||||
from requests import HTTPError, request
|
||||
|
||||
from app.celery.process_ses_receipts_tasks import process_ses_results
|
||||
from app.config import QueueNames
|
||||
from app.dao.notifications_dao import get_notification_by_id
|
||||
from app.enums import NotificationType
|
||||
|
||||
temp_fail = "2028675303"
|
||||
perm_fail = "2028675302"
|
||||
delivered = "2028675309"
|
||||
|
||||
delivered_email = "delivered@simulator.notify"
|
||||
perm_fail_email = "perm-fail@simulator.notify"
|
||||
temp_fail_email = "temp-fail@simulator.notify"
|
||||
|
||||
|
||||
def send_sms_response(provider, reference):
|
||||
body = sns_callback(reference)
|
||||
headers = {"Content-type": "application/json"}
|
||||
|
||||
make_request(NotificationType.SMS, provider, body, headers)
|
||||
|
||||
|
||||
def send_email_response(reference, to):
|
||||
if to == perm_fail_email:
|
||||
body = ses_hard_bounce_callback(reference)
|
||||
elif to == temp_fail_email:
|
||||
body = ses_soft_bounce_callback(reference)
|
||||
else:
|
||||
body = ses_notification_callback(reference)
|
||||
|
||||
process_ses_results.apply_async([body], queue=QueueNames.SEND_EMAIL)
|
||||
|
||||
|
||||
def make_request(notification_type, provider, data, headers):
|
||||
api_call = "{}/notifications/{}/{}".format(
|
||||
current_app.config["API_HOST_NAME"], notification_type, provider
|
||||
)
|
||||
|
||||
try:
|
||||
response = request("POST", api_call, headers=headers, data=data, timeout=60)
|
||||
response.raise_for_status()
|
||||
except HTTPError as e:
|
||||
current_app.logger.error(
|
||||
"API POST request on {} failed with status {}".format(
|
||||
api_call, e.response.status_code
|
||||
)
|
||||
)
|
||||
raise e
|
||||
finally:
|
||||
current_app.logger.info("Mocked provider callback request finished")
|
||||
return response.json()
|
||||
|
||||
|
||||
def sns_callback(notification_id):
|
||||
notification = get_notification_by_id(notification_id)
|
||||
|
||||
# This will only work if all notifications, including successful ones, are in the notifications table
|
||||
# If we decide to delete successful notifications, we will have to get this from notifications history
|
||||
return json.dumps(
|
||||
{
|
||||
"CID": str(notification_id),
|
||||
"status": notification.status,
|
||||
# "deliverytime": notification.completed_at
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def ses_notification_callback(reference):
|
||||
ses_message_body = {
|
||||
"delivery": {
|
||||
"processingTimeMillis": 2003,
|
||||
"recipients": ["success@simulator.amazonses.com"],
|
||||
"remoteMtaIp": "123.123.123.123",
|
||||
"reportingMTA": "a7-32.smtp-out.us-west-2.amazonses.com",
|
||||
"smtpResponse": "250 2.6.0 Message received",
|
||||
"timestamp": "2017-11-17T12:14:03.646Z",
|
||||
},
|
||||
"mail": {
|
||||
"commonHeaders": {
|
||||
"from": ["TEST <TEST@notify.works>"],
|
||||
"subject": "lambda test",
|
||||
"to": ["success@simulator.amazonses.com"],
|
||||
},
|
||||
"destination": ["success@simulator.amazonses.com"],
|
||||
"headers": [
|
||||
{"name": "From", "value": "TEST <TEST@notify.works>"},
|
||||
{"name": "To", "value": "success@simulator.amazonses.com"},
|
||||
{"name": "Subject", "value": "lambda test"},
|
||||
{"name": "MIME-Version", "value": "1.0"},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": 'multipart/alternative; boundary="----=_Part_617203_1627511946.1510920841645"',
|
||||
},
|
||||
],
|
||||
"headersTruncated": False,
|
||||
"messageId": reference,
|
||||
"sendingAccountId": "12341234",
|
||||
"source": '"TEST" <TEST@notify.works>',
|
||||
"sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
|
||||
"sourceIp": "0.0.0.1",
|
||||
"timestamp": "2017-11-17T12:14:01.643Z",
|
||||
},
|
||||
"notificationType": "Delivery",
|
||||
}
|
||||
|
||||
return {
|
||||
"Type": "Notification",
|
||||
"MessageId": "8e83c020-1234-1234-1234-92a8ee9baa0a",
|
||||
"TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
|
||||
"Subject": None,
|
||||
"Message": json.dumps(ses_message_body),
|
||||
"Timestamp": "2017-11-17T12:14:03.710Z",
|
||||
"SignatureVersion": "1",
|
||||
"Signature": "[REDACTED]",
|
||||
"SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem",
|
||||
"UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]",
|
||||
"MessageAttributes": {},
|
||||
}
|
||||
|
||||
|
||||
def ses_hard_bounce_callback(reference):
|
||||
return _ses_bounce_callback(reference, "Permanent")
|
||||
|
||||
|
||||
def ses_soft_bounce_callback(reference):
|
||||
return _ses_bounce_callback(reference, "Temporary")
|
||||
|
||||
|
||||
def _ses_bounce_callback(reference, bounce_type):
|
||||
ses_message_body = {
|
||||
"bounce": {
|
||||
"bounceSubType": "General",
|
||||
"bounceType": bounce_type,
|
||||
"bouncedRecipients": [
|
||||
{
|
||||
"action": "failed",
|
||||
"diagnosticCode": "smtp; 550 5.1.1 user unknown",
|
||||
"emailAddress": "bounce@simulator.amazonses.com",
|
||||
"status": "5.1.1",
|
||||
}
|
||||
],
|
||||
"feedbackId": "0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000",
|
||||
"remoteMtaIp": "123.123.123.123",
|
||||
"reportingMTA": "dsn; a7-31.smtp-out.us-west-2.amazonses.com",
|
||||
"timestamp": "2017-11-17T12:14:05.131Z",
|
||||
},
|
||||
"mail": {
|
||||
"commonHeaders": {
|
||||
"from": ["TEST <TEST@notify.works>"],
|
||||
"subject": "ses callback test",
|
||||
"to": ["bounce@simulator.amazonses.com"],
|
||||
},
|
||||
"destination": ["bounce@simulator.amazonses.com"],
|
||||
"headers": [
|
||||
{"name": "From", "value": "TEST <TEST@notify.works>"},
|
||||
{"name": "To", "value": "bounce@simulator.amazonses.com"},
|
||||
{"name": "Subject", "value": "lambda test"},
|
||||
{"name": "MIME-Version", "value": "1.0"},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": 'multipart/alternative; boundary="----=_Part_596529_2039165601.1510920843367"',
|
||||
},
|
||||
],
|
||||
"headersTruncated": False,
|
||||
"messageId": reference,
|
||||
"sendingAccountId": "12341234",
|
||||
"source": '"TEST" <TEST@notify.works>',
|
||||
"sourceArn": "arn:aws:ses:us-west-2:12341234:identity/notify.works",
|
||||
"sourceIp": "0.0.0.1",
|
||||
"timestamp": "2017-11-17T12:14:03.000Z",
|
||||
},
|
||||
"notificationType": "Bounce",
|
||||
}
|
||||
return {
|
||||
"Type": "Notification",
|
||||
"MessageId": "36e67c28-1234-1234-1234-2ea0172aa4a7",
|
||||
"TopicArn": "arn:aws:sns:us-west-2:12341234:ses_notifications",
|
||||
"Subject": None,
|
||||
"Message": json.dumps(ses_message_body),
|
||||
"Timestamp": "2017-11-17T12:14:05.149Z",
|
||||
"SignatureVersion": "1",
|
||||
"Signature": "[REDACTED]", # noqa
|
||||
"SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem",
|
||||
"UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]",
|
||||
"MessageAttributes": {},
|
||||
}
|
||||
@@ -1,41 +1,23 @@
|
||||
from abc import abstractmethod
|
||||
from typing import Protocol
|
||||
|
||||
from botocore.config import Config
|
||||
|
||||
from app.enums import NotificationType
|
||||
|
||||
AWS_CLIENT_CONFIG = Config(
|
||||
# This config is required to enable S3 to connect to FIPS-enabled
|
||||
# endpoints. See https://aws.amazon.com/compliance/fips/ for more
|
||||
# information.
|
||||
s3={
|
||||
"addressing_style": "virtual",
|
||||
},
|
||||
use_fips_endpoint=True,
|
||||
max_pool_connections=50, # This should be equal or greater than our celery concurrency
|
||||
)
|
||||
|
||||
|
||||
class ClientException(Exception):
|
||||
"""
|
||||
'''
|
||||
Base Exceptions for sending notifications that fail
|
||||
"""
|
||||
|
||||
'''
|
||||
pass
|
||||
|
||||
|
||||
class Client(Protocol):
|
||||
"""
|
||||
class Client(object):
|
||||
'''
|
||||
Base client for sending notifications.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def init_app(self, current_app, *args, **kwargs):
|
||||
raise NotImplementedError("TODO: Need to implement.")
|
||||
'''
|
||||
pass
|
||||
|
||||
|
||||
class NotificationProviderClients(object):
|
||||
STATISTICS_REQUESTED = 'requested'
|
||||
STATISTICS_DELIVERED = 'delivered'
|
||||
STATISTICS_FAILURE = 'failure'
|
||||
|
||||
|
||||
class Clients(object):
|
||||
sms_clients = {}
|
||||
email_clients = {}
|
||||
|
||||
@@ -53,13 +35,10 @@ class NotificationProviderClients(object):
|
||||
return self.email_clients.get(name)
|
||||
|
||||
def get_client_by_name_and_type(self, name, notification_type):
|
||||
assert notification_type in {
|
||||
NotificationType.EMAIL,
|
||||
NotificationType.SMS,
|
||||
} # nosec B101
|
||||
assert notification_type in ['email', 'sms']
|
||||
|
||||
if notification_type == NotificationType.EMAIL:
|
||||
if notification_type == 'email':
|
||||
return self.get_email_client(name)
|
||||
|
||||
if notification_type == NotificationType.SMS:
|
||||
if notification_type == 'sms':
|
||||
return self.get_sms_client(name)
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from boto3 import client
|
||||
from flask import current_app
|
||||
|
||||
from app.clients import AWS_CLIENT_CONFIG, Client
|
||||
from app.cloudfoundry_config import cloud_config
|
||||
|
||||
|
||||
class AwsCloudwatchClient(Client):
|
||||
"""
|
||||
This client is responsible for retrieving sms delivery receipts from cloudwatch.
|
||||
"""
|
||||
|
||||
def init_app(self, current_app, *args, **kwargs):
|
||||
if os.getenv("LOCALSTACK_ENDPOINT_URL"):
|
||||
self._client = client(
|
||||
"logs",
|
||||
region_name=cloud_config.sns_region,
|
||||
aws_access_key_id=cloud_config.sns_access_key,
|
||||
aws_secret_access_key=cloud_config.sns_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
endpoint_url=os.getenv("LOCALSTACK_ENDPOINT_URL"),
|
||||
)
|
||||
self._is_localstack = True
|
||||
else:
|
||||
self._client = client(
|
||||
"logs",
|
||||
region_name=cloud_config.sns_region,
|
||||
aws_access_key_id=cloud_config.sns_access_key,
|
||||
aws_secret_access_key=cloud_config.sns_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
)
|
||||
self._is_localstack = False
|
||||
|
||||
super(Client, self).__init__(*args, **kwargs)
|
||||
self.current_app = current_app
|
||||
self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$")
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "cloudwatch"
|
||||
|
||||
def is_localstack(self):
|
||||
return self._is_localstack
|
||||
|
||||
def _get_log(self, log_group_name, start, end):
|
||||
# Check all cloudwatch logs from the time the notification was sent (currently 5 minutes previously) until now
|
||||
next_token = None
|
||||
all_log_events = []
|
||||
|
||||
while True:
|
||||
if next_token:
|
||||
response = self._client.filter_log_events(
|
||||
logGroupName=log_group_name,
|
||||
nextToken=next_token,
|
||||
startTime=int(start.timestamp() * 1000),
|
||||
endTime=int(end.timestamp() * 1000),
|
||||
)
|
||||
else:
|
||||
response = self._client.filter_log_events(
|
||||
logGroupName=log_group_name,
|
||||
startTime=int(start.timestamp() * 1000),
|
||||
endTime=int(end.timestamp() * 1000),
|
||||
)
|
||||
log_events = response.get("events", [])
|
||||
all_log_events.extend(log_events)
|
||||
next_token = response.get("nextToken")
|
||||
if not next_token:
|
||||
break
|
||||
return all_log_events
|
||||
|
||||
def warn_if_dev_is_opted_out(self, provider_response, notification_id):
|
||||
if (
|
||||
"is opted out" in provider_response.lower()
|
||||
or "has blocked sms" in provider_response.lower()
|
||||
):
|
||||
if os.getenv("NOTIFY_ENVIRONMENT") in ["development", "test"]:
|
||||
ansi_red = "\033[31m"
|
||||
ansi_reset = "\033[0m"
|
||||
logline = (
|
||||
ansi_red
|
||||
+ f"The phone number for notification_id {notification_id} is OPTED OUT. You need to opt back in"
|
||||
+ ansi_reset
|
||||
)
|
||||
current_app.logger.warning(logline)
|
||||
return logline
|
||||
return None
|
||||
|
||||
def _extract_account_number(self, ses_domain_arn):
|
||||
account_number = ses_domain_arn.split(":")
|
||||
return account_number
|
||||
|
||||
def event_to_db_format(self, event):
|
||||
|
||||
# massage the data into the form the db expects. When we switch
|
||||
# from filter_log_events to log insights this will be convenient
|
||||
if isinstance(event, str):
|
||||
event = json.loads(event)
|
||||
|
||||
# Don't trust AWS to always send the same JSON structure back
|
||||
# However, if we don't get message_id and status we might as well blow up
|
||||
# because it's pointless to continue
|
||||
phone_carrier = self._aws_value_or_default(event, "delivery", "phoneCarrier")
|
||||
provider_response = self._aws_value_or_default(
|
||||
event, "delivery", "providerResponse"
|
||||
)
|
||||
message_cost = self._aws_value_or_default(event, "delivery", "priceInUSD")
|
||||
if message_cost is None or message_cost == "":
|
||||
message_cost = 0.0
|
||||
else:
|
||||
message_cost = float(message_cost)
|
||||
|
||||
my_timestamp = self._aws_value_or_default(event, "notification", "timestamp")
|
||||
return {
|
||||
"notification.messageId": event["notification"]["messageId"],
|
||||
"status": event["status"],
|
||||
"delivery.phoneCarrier": phone_carrier,
|
||||
"delivery.providerResponse": provider_response,
|
||||
"@timestamp": my_timestamp,
|
||||
"delivery.priceInUSD": message_cost,
|
||||
}
|
||||
|
||||
# Here is an example of how to get the events with log insights
|
||||
# def do_log_insights():
|
||||
# query = """
|
||||
# fields @timestamp, status, message, recipient
|
||||
# | filter status = "DELIVERED"
|
||||
# | sort @timestamp asc
|
||||
# """
|
||||
# temp_client = boto3.client(
|
||||
# "logs",
|
||||
# region_name="us-gov-west-1",
|
||||
# aws_access_key_id=AWS_ACCESS_KEY_ID,
|
||||
# aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
||||
# config=AWS_CLIENT_CONFIG,
|
||||
# )
|
||||
# start = utc_now()
|
||||
# end = utc_now - timedelta(hours=1)
|
||||
# response = temp_client.start_query(
|
||||
# logGroupName = LOG_GROUP_NAME_DELIVERED,
|
||||
# startTime = int(start.timestamp()),
|
||||
# endTime= int(end.timestamp()),
|
||||
# queryString = query
|
||||
|
||||
# )
|
||||
# query_id = response['queryId']
|
||||
# while True:
|
||||
# result = temp_client.get_query_results(queryId=query_id)
|
||||
# if result['status'] == 'Complete':
|
||||
# break
|
||||
# eventlet.sleep(1)
|
||||
|
||||
# delivery_receipts = []
|
||||
# for log in result['results']:
|
||||
# receipt = {field['field']: field['value'] for field in log}
|
||||
# delivery_receipts.append(receipt)
|
||||
# print(receipt)
|
||||
|
||||
# print(len(delivery_receipts))
|
||||
|
||||
# In the long run we want to use Log Insights because it is more efficient
|
||||
# that filter_log_events. But we are blocked by a permissions issue in the broker.
|
||||
# So for now, use filter_log_events and grab all log_events over a 10 minute interval,
|
||||
# and run this on a schedule.
|
||||
def check_delivery_receipts(self, start, end):
|
||||
region = cloud_config.sns_region
|
||||
account_number = self._extract_account_number(cloud_config.ses_domain_arn)
|
||||
log_group_name = f"sns/{region}/{account_number[4]}/DirectPublishToPhoneNumber"
|
||||
delivered_event_set = self._get_receipts(log_group_name, start, end)
|
||||
current_app.logger.info(
|
||||
(f"Delivered message count: {len(delivered_event_set)}")
|
||||
)
|
||||
log_group_name = (
|
||||
f"sns/{region}/{account_number[4]}/DirectPublishToPhoneNumber/Failure"
|
||||
)
|
||||
failed_event_set = self._get_receipts(log_group_name, start, end)
|
||||
current_app.logger.info((f"Failed message count: {len(failed_event_set)}"))
|
||||
|
||||
return delivered_event_set, failed_event_set
|
||||
|
||||
def _get_receipts(self, log_group_name, start, end):
|
||||
event_set = set()
|
||||
all_events = self._get_log(log_group_name, start, end)
|
||||
for event in all_events:
|
||||
try:
|
||||
actual_event = self.event_to_db_format(event["message"])
|
||||
event_set.add(json.dumps(actual_event))
|
||||
except Exception:
|
||||
current_app.logger.exception(
|
||||
f"Could not format delivery receipt {event} for db insert"
|
||||
)
|
||||
return event_set
|
||||
|
||||
def _aws_value_or_default(self, event, top_level, second_level):
|
||||
if event.get(top_level) is None or event[top_level].get(second_level) is None:
|
||||
my_var = ""
|
||||
else:
|
||||
my_var = event[top_level][second_level]
|
||||
|
||||
return my_var
|
||||
@@ -1,4 +1,7 @@
|
||||
import base64
|
||||
|
||||
import requests
|
||||
|
||||
from flask import current_app
|
||||
|
||||
|
||||
@@ -9,49 +12,44 @@ class DocumentDownloadError(Exception):
|
||||
|
||||
@classmethod
|
||||
def from_exception(cls, e):
|
||||
message = e.response.json()["error"]
|
||||
status_code = e.response.status_code
|
||||
try:
|
||||
message = e.response.json()['error']
|
||||
status_code = e.response.status_code
|
||||
except (TypeError, ValueError, AttributeError, KeyError):
|
||||
message = 'connection error'
|
||||
status_code = 503
|
||||
|
||||
return cls(message, status_code)
|
||||
|
||||
|
||||
class DocumentDownloadClient:
|
||||
|
||||
def init_app(self, app):
|
||||
self.api_host = app.config["DOCUMENT_DOWNLOAD_API_HOST"]
|
||||
self.auth_token = app.config["DOCUMENT_DOWNLOAD_API_KEY"]
|
||||
self.api_host = app.config['DOCUMENT_DOWNLOAD_API_HOST']
|
||||
self.auth_token = app.config['DOCUMENT_DOWNLOAD_API_KEY']
|
||||
|
||||
def get_upload_url(self, service_id):
|
||||
return "{}/services/{}/documents".format(self.api_host, service_id)
|
||||
|
||||
def upload_document(self, service_id, file_contents, is_csv=None):
|
||||
def upload_document(self, service_id, file_contents):
|
||||
try:
|
||||
response = requests.post(
|
||||
self.get_upload_url(service_id),
|
||||
headers={
|
||||
"Authorization": "Bearer {}".format(self.auth_token),
|
||||
'Authorization': "Bearer {}".format(self.auth_token),
|
||||
},
|
||||
json={
|
||||
"document": file_contents,
|
||||
"is_csv": is_csv or False,
|
||||
},
|
||||
timeout=30,
|
||||
files={
|
||||
'document': base64.b64decode(file_contents)
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
# if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
|
||||
# we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
|
||||
# unrecognised) should be raised as a 500 internal server error here.
|
||||
if e.response is None:
|
||||
raise Exception(f"Unhandled document download error: {repr(e)}")
|
||||
elif e.response.status_code == 400:
|
||||
error = DocumentDownloadError.from_exception(e)
|
||||
current_app.logger.info(
|
||||
"Document download request failed with error: {}".format(
|
||||
error.message
|
||||
)
|
||||
)
|
||||
raise error
|
||||
else:
|
||||
raise Exception(f"Unhandled document download error: {e.response.text}")
|
||||
error = DocumentDownloadError.from_exception(e)
|
||||
current_app.logger.warning(
|
||||
'Document download request failed with error: {}'.format(error.message)
|
||||
)
|
||||
|
||||
return response.json()["document"]["url"]
|
||||
raise error
|
||||
|
||||
return response.json()['document']['url']
|
||||
|
||||
@@ -1,38 +1,20 @@
|
||||
from abc import abstractmethod, abstractproperty
|
||||
|
||||
from app.clients import Client, ClientException
|
||||
from app.clients import ClientException, Client
|
||||
|
||||
|
||||
class EmailClientException(ClientException):
|
||||
"""
|
||||
'''
|
||||
Base Exception for EmailClients
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class EmailClientNonRetryableException(ClientException):
|
||||
"""
|
||||
Represents an error returned from the email client API with a 4xx response code
|
||||
that should not be retried and should instead be marked as technical failure.
|
||||
An example of this would be an email address that makes it through our
|
||||
validation rules but is rejected by SES. There is no point in retrying this type as
|
||||
it will always fail however many calls to SES. Whereas a throttling error would not
|
||||
use this exception as it may succeed if we retry
|
||||
"""
|
||||
|
||||
'''
|
||||
pass
|
||||
|
||||
|
||||
class EmailClient(Client):
|
||||
"""
|
||||
'''
|
||||
Base Email client for sending emails.
|
||||
"""
|
||||
'''
|
||||
|
||||
@abstractmethod
|
||||
def send_email(self, *args, **kwargs):
|
||||
raise NotImplementedError("TODO Need to implement.")
|
||||
raise NotImplementedError('TODO Need to implement.')
|
||||
|
||||
@abstractproperty
|
||||
def name(self):
|
||||
raise NotImplementedError("TODO Need to implement.")
|
||||
def get_name(self):
|
||||
raise NotImplementedError('TODO Need to implement.')
|
||||
|
||||
@@ -1,43 +1,37 @@
|
||||
from time import monotonic
|
||||
|
||||
import boto3
|
||||
import botocore
|
||||
from boto3 import client
|
||||
from flask import current_app
|
||||
from time import monotonic
|
||||
from notifications_utils.recipients import InvalidEmailError
|
||||
|
||||
from app.clients import AWS_CLIENT_CONFIG
|
||||
from app.clients.email import (
|
||||
EmailClient,
|
||||
EmailClientException,
|
||||
EmailClientNonRetryableException,
|
||||
)
|
||||
from app.cloudfoundry_config import cloud_config
|
||||
from app.enums import NotificationStatus, StatisticsType
|
||||
from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
|
||||
from app.clients.email import (EmailClientException, EmailClient)
|
||||
|
||||
ses_response_map = {
|
||||
"Permanent": {
|
||||
"message": "Hard bounced",
|
||||
'Permanent': {
|
||||
"message": 'Hard bounced',
|
||||
"success": False,
|
||||
"notification_status": NotificationStatus.PERMANENT_FAILURE,
|
||||
"notification_statistics_status": StatisticsType.FAILURE,
|
||||
"notification_status": 'permanent-failure',
|
||||
"notification_statistics_status": STATISTICS_FAILURE
|
||||
},
|
||||
"Temporary": {
|
||||
"message": "Soft bounced",
|
||||
'Temporary': {
|
||||
"message": 'Soft bounced',
|
||||
"success": False,
|
||||
"notification_status": NotificationStatus.TEMPORARY_FAILURE,
|
||||
"notification_statistics_status": StatisticsType.FAILURE,
|
||||
"notification_status": 'temporary-failure',
|
||||
"notification_statistics_status": STATISTICS_FAILURE
|
||||
},
|
||||
"Delivery": {
|
||||
"message": "Delivered",
|
||||
'Delivery': {
|
||||
"message": 'Delivered',
|
||||
"success": True,
|
||||
"notification_status": NotificationStatus.DELIVERED,
|
||||
"notification_statistics_status": StatisticsType.DELIVERED,
|
||||
"notification_status": 'delivered',
|
||||
"notification_statistics_status": STATISTICS_DELIVERED
|
||||
},
|
||||
"Complaint": {
|
||||
"message": "Complaint",
|
||||
'Complaint': {
|
||||
"message": 'Complaint',
|
||||
"success": True,
|
||||
"notification_status": NotificationStatus.DELIVERED,
|
||||
"notification_statistics_status": StatisticsType.DELIVERED,
|
||||
},
|
||||
"notification_status": 'delivered',
|
||||
"notification_statistics_status": STATISTICS_DELIVERED
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,90 +43,82 @@ class AwsSesClientException(EmailClientException):
|
||||
pass
|
||||
|
||||
|
||||
class AwsSesClientThrottlingSendRateException(AwsSesClientException):
|
||||
pass
|
||||
|
||||
|
||||
class AwsSesClient(EmailClient):
|
||||
"""
|
||||
'''
|
||||
Amazon SES email client.
|
||||
"""
|
||||
'''
|
||||
|
||||
def init_app(self, *args, **kwargs):
|
||||
self._client = client(
|
||||
"ses",
|
||||
region_name=cloud_config.ses_region,
|
||||
aws_access_key_id=cloud_config.ses_access_key,
|
||||
aws_secret_access_key=cloud_config.ses_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
)
|
||||
def init_app(self, region, statsd_client, *args, **kwargs):
|
||||
self._client = boto3.client('ses', region_name=region)
|
||||
super(AwsSesClient, self).__init__(*args, **kwargs)
|
||||
self.name = 'ses'
|
||||
self.statsd_client = statsd_client
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "ses"
|
||||
def get_name(self):
|
||||
return self.name
|
||||
|
||||
def send_email(
|
||||
self, source, to_addresses, subject, body, html_body="", reply_to_address=None
|
||||
):
|
||||
def send_email(self,
|
||||
source,
|
||||
to_addresses,
|
||||
subject,
|
||||
body,
|
||||
html_body='',
|
||||
reply_to_address=None):
|
||||
try:
|
||||
if isinstance(to_addresses, str):
|
||||
to_addresses = [to_addresses]
|
||||
|
||||
reply_to_addresses = [reply_to_address] if reply_to_address else []
|
||||
|
||||
body = {"Text": {"Data": body}}
|
||||
body = {
|
||||
'Text': {'Data': body}
|
||||
}
|
||||
|
||||
if html_body:
|
||||
body.update({"Html": {"Data": html_body}})
|
||||
body.update({
|
||||
'Html': {'Data': html_body}
|
||||
})
|
||||
|
||||
start_time = monotonic()
|
||||
response = self._client.send_email(
|
||||
Source=source,
|
||||
Destination={
|
||||
"ToAddresses": [
|
||||
punycode_encode_email(addr) for addr in to_addresses
|
||||
],
|
||||
"CcAddresses": [],
|
||||
"BccAddresses": [],
|
||||
'ToAddresses': [punycode_encode_email(addr) for addr in to_addresses],
|
||||
'CcAddresses': [],
|
||||
'BccAddresses': []
|
||||
},
|
||||
Message={
|
||||
"Subject": {
|
||||
"Data": subject,
|
||||
'Subject': {
|
||||
'Data': subject,
|
||||
},
|
||||
"Body": body,
|
||||
'Body': body
|
||||
},
|
||||
ReplyToAddresses=[
|
||||
punycode_encode_email(addr) for addr in reply_to_addresses
|
||||
],
|
||||
ReplyToAddresses=[punycode_encode_email(addr) for addr in reply_to_addresses]
|
||||
)
|
||||
except botocore.exceptions.ClientError as e:
|
||||
_do_fancy_exception_handling(e)
|
||||
self.statsd_client.incr("clients.ses.error")
|
||||
|
||||
# http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
|
||||
if e.response['Error']['Code'] == 'InvalidParameterValue':
|
||||
raise InvalidEmailError('email: "{}" message: "{}"'.format(
|
||||
to_addresses[0],
|
||||
e.response['Error']['Message']
|
||||
))
|
||||
else:
|
||||
self.statsd_client.incr("clients.ses.error")
|
||||
raise AwsSesClientException(str(e))
|
||||
except Exception as e:
|
||||
self.statsd_client.incr("clients.ses.error")
|
||||
raise AwsSesClientException(str(e))
|
||||
else:
|
||||
elapsed_time = monotonic() - start_time
|
||||
current_app.logger.info(
|
||||
"AWS SES request finished in {}".format(elapsed_time)
|
||||
)
|
||||
return response["MessageId"]
|
||||
current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
|
||||
self.statsd_client.timing("clients.ses.request-time", elapsed_time)
|
||||
self.statsd_client.incr("clients.ses.success")
|
||||
return response['MessageId']
|
||||
|
||||
|
||||
def punycode_encode_email(email_address):
|
||||
# only the hostname should ever be punycode encoded.
|
||||
local, hostname = email_address.split("@")
|
||||
return "{}@{}".format(local, hostname.encode("idna").decode("utf-8"))
|
||||
|
||||
|
||||
def _do_fancy_exception_handling(e):
|
||||
# http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
|
||||
if e.response["Error"]["Code"] == "InvalidParameterValue":
|
||||
raise EmailClientNonRetryableException(e.response["Error"]["Message"])
|
||||
elif (
|
||||
e.response["Error"]["Code"] == "Throttling"
|
||||
and e.response["Error"]["Message"] == "Maximum sending rate exceeded."
|
||||
):
|
||||
raise AwsSesClientThrottlingSendRateException(str(e))
|
||||
else:
|
||||
raise AwsSesClientException(str(e))
|
||||
local, hostname = email_address.split('@')
|
||||
return '{}@{}'.format(local, hostname.encode('idna').decode('utf-8'))
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
import json
|
||||
from time import monotonic
|
||||
|
||||
from flask import current_app
|
||||
from requests import request
|
||||
|
||||
from app.clients.email import EmailClient, EmailClientException
|
||||
|
||||
|
||||
class AwsSesStubClientException(EmailClientException):
|
||||
pass
|
||||
|
||||
|
||||
class AwsSesStubClient(EmailClient):
|
||||
def init_app(self, stub_url):
|
||||
self.url = stub_url
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "ses"
|
||||
|
||||
def send_email(
|
||||
self, source, to_addresses, subject, body, html_body="", reply_to_address=None
|
||||
):
|
||||
try:
|
||||
start_time = monotonic()
|
||||
response = request("POST", self.url, data={"id": "dummy-data"}, timeout=60)
|
||||
response.raise_for_status()
|
||||
response_json = json.loads(response.text)
|
||||
|
||||
except Exception as e:
|
||||
raise AwsSesStubClientException(str(e))
|
||||
else:
|
||||
elapsed_time = monotonic() - start_time
|
||||
current_app.logger.info(
|
||||
"AWS SES stub request finished in {}".format(elapsed_time)
|
||||
)
|
||||
return response_json["MessageId"]
|
||||
@@ -1,55 +1,52 @@
|
||||
import base64
|
||||
import json
|
||||
|
||||
import requests
|
||||
from flask import current_app
|
||||
import requests
|
||||
|
||||
from notifications_utils.timezones import convert_utc_to_bst
|
||||
|
||||
|
||||
class PerformancePlatformClient:
|
||||
|
||||
@property
|
||||
def active(self):
|
||||
return self._active
|
||||
|
||||
def init_app(self, app, *args, **kwargs):
|
||||
self._active = app.config.get("PERFORMANCE_PLATFORM_ENABLED")
|
||||
def init_app(self, app):
|
||||
self._active = app.config.get('PERFORMANCE_PLATFORM_ENABLED')
|
||||
if self.active:
|
||||
self.performance_platform_url = app.config.get("PERFORMANCE_PLATFORM_URL")
|
||||
self.performance_platform_endpoints = app.config.get(
|
||||
"PERFORMANCE_PLATFORM_ENDPOINTS"
|
||||
)
|
||||
self.performance_platform_url = app.config.get('PERFORMANCE_PLATFORM_URL')
|
||||
self.performance_platform_endpoints = app.config.get('PERFORMANCE_PLATFORM_ENDPOINTS')
|
||||
|
||||
def send_stats_to_performance_platform(self, payload):
|
||||
if self.active:
|
||||
bearer_token = self.performance_platform_endpoints[payload["dataType"]]
|
||||
bearer_token = self.performance_platform_endpoints[payload['dataType']]
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer {}".format(bearer_token),
|
||||
'Content-Type': "application/json",
|
||||
'Authorization': 'Bearer {}'.format(bearer_token)
|
||||
}
|
||||
resp = requests.post(
|
||||
self.performance_platform_url + payload["dataType"],
|
||||
self.performance_platform_url + payload['dataType'],
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
headers=headers
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
current_app.logger.info(
|
||||
"Updated performance platform successfully with payload {}".format(
|
||||
json.dumps(payload)
|
||||
)
|
||||
"Updated performance platform successfully with payload {}".format(json.dumps(payload))
|
||||
)
|
||||
else:
|
||||
current_app.logger.error(
|
||||
"Performance platform update request failed for payload with response details: {} '{}'".format(
|
||||
json.dumps(payload), resp.status_code
|
||||
json.dumps(payload),
|
||||
resp.status_code
|
||||
)
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
@staticmethod
|
||||
def format_payload(
|
||||
*, dataset, start_time, group_name, group_value, count, period="day"
|
||||
):
|
||||
def format_payload(*, dataset, start_time, group_name, group_value, count, period='day'):
|
||||
"""
|
||||
:param dataset - the name of the overall graph, as referred to in the endpoint.
|
||||
:param start_time - UTC midnight of the day we're sending stats for
|
||||
@@ -59,16 +56,14 @@ class PerformancePlatformClient:
|
||||
:param period - the period that this data covers - "day", "week", "month", "quarter".
|
||||
"""
|
||||
payload = {
|
||||
"_timestamp": start_time,
|
||||
"service": "govuk-notify",
|
||||
"dataType": dataset,
|
||||
"period": period,
|
||||
"count": count,
|
||||
'_timestamp': convert_utc_to_bst(start_time).isoformat(),
|
||||
'service': 'govuk-notify',
|
||||
'dataType': dataset,
|
||||
'period': period,
|
||||
'count': count,
|
||||
group_name: group_value,
|
||||
}
|
||||
payload["_id"] = PerformancePlatformClient.generate_payload_id(
|
||||
payload, group_name
|
||||
)
|
||||
payload['_id'] = PerformancePlatformClient.generate_payload_id(payload, group_name)
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
@@ -76,12 +71,12 @@ class PerformancePlatformClient:
|
||||
"""
|
||||
group_name is the name of the group - eg "channel" or "status"
|
||||
"""
|
||||
payload_string = "{}{}{}{}{}".format(
|
||||
payload["_timestamp"],
|
||||
payload["service"],
|
||||
payload_string = '{}{}{}{}{}'.format(
|
||||
payload['_timestamp'],
|
||||
payload['service'],
|
||||
payload[group_name],
|
||||
payload["dataType"],
|
||||
payload["period"],
|
||||
payload['dataType'],
|
||||
payload['period']
|
||||
)
|
||||
_id = base64.b64encode(payload_string.encode("utf-8"))
|
||||
return _id.decode("utf-8")
|
||||
_id = base64.b64encode(payload_string.encode('utf-8'))
|
||||
return _id.decode('utf-8')
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
from boto3 import client
|
||||
from botocore.exceptions import ClientError
|
||||
from flask import current_app
|
||||
|
||||
from app.clients import AWS_CLIENT_CONFIG, Client
|
||||
from app.cloudfoundry_config import cloud_config
|
||||
from app.utils import hilite
|
||||
|
||||
|
||||
class AwsPinpointClient(Client):
|
||||
|
||||
def init_app(self, current_app, *args, **kwargs):
|
||||
self._client = client(
|
||||
"pinpoint",
|
||||
region_name=cloud_config.sns_region,
|
||||
aws_access_key_id=cloud_config.sns_access_key,
|
||||
aws_secret_access_key=cloud_config.sns_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
)
|
||||
|
||||
super(Client, self).__init__(*args, **kwargs)
|
||||
self.current_app = current_app
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "pinpoint"
|
||||
|
||||
def validate_phone_number(self, country_code, phone_number):
|
||||
try:
|
||||
response = self._client.phone_number_validate(
|
||||
NumberValidateRequest={
|
||||
"IsoCountryCode": country_code,
|
||||
"PhoneNumber": phone_number,
|
||||
}
|
||||
)
|
||||
|
||||
# TODO right now this will only print with AWS simulated numbers,
|
||||
# but remove this when that changes
|
||||
current_app.logger.info(hilite(response))
|
||||
return response
|
||||
except ClientError:
|
||||
current_app.logger.exception(
|
||||
"#notify-debug-validate-phone-number Could not validate with pinpoint"
|
||||
)
|
||||
|
||||
# TODO This is the structure of the response. When the phone validation
|
||||
# capability we want to offer is better defined (it may just be a question
|
||||
# of checking PhoneType -- i.e., landline or mobile) then do something with
|
||||
# this info.
|
||||
# {
|
||||
# 'NumberValidateResponse': {
|
||||
# 'Carrier': 'string',
|
||||
# 'City': 'string',
|
||||
# 'CleansedPhoneNumberE164': 'string',
|
||||
# 'CleansedPhoneNumberNational': 'string',
|
||||
# 'Country': 'string',
|
||||
# 'CountryCodeIso2': 'string',
|
||||
# 'CountryCodeNumeric': 'string',
|
||||
# 'County': 'string',
|
||||
# 'OriginalCountryCodeIso2': 'string',
|
||||
# 'OriginalPhoneNumber': 'string',
|
||||
# 'PhoneType': 'string',
|
||||
# 'PhoneTypeCode': 123,
|
||||
# 'Timezone': 'string',
|
||||
# 'ZipCode': 'string'
|
||||
# }
|
||||
# }
|
||||
@@ -1,34 +1,25 @@
|
||||
from abc import abstractmethod, abstractproperty
|
||||
from typing import final
|
||||
|
||||
from app.clients import Client, ClientException
|
||||
from app.clients import (Client, ClientException)
|
||||
|
||||
|
||||
class SmsClientResponseException(ClientException):
|
||||
"""
|
||||
'''
|
||||
Base Exception for SmsClientsResponses
|
||||
"""
|
||||
'''
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
def __str__(self):
|
||||
return f"Message {self.message}"
|
||||
return "Message {}".format(self.message)
|
||||
|
||||
|
||||
class SmsClient(Client):
|
||||
"""
|
||||
'''
|
||||
Base Sms client for sending smss.
|
||||
"""
|
||||
'''
|
||||
|
||||
@abstractmethod
|
||||
def send_sms(self, *args, **kwargs):
|
||||
raise NotImplementedError("TODO Need to implement.")
|
||||
raise NotImplementedError('TODO Need to implement.')
|
||||
|
||||
@abstractproperty
|
||||
def name(self):
|
||||
raise NotImplementedError("TODO Need to implement.")
|
||||
|
||||
@final
|
||||
def get_name(self):
|
||||
return self.name
|
||||
raise NotImplementedError('TODO Need to implement.')
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
from time import monotonic
|
||||
|
||||
import botocore
|
||||
import phonenumbers
|
||||
from boto3 import client
|
||||
|
||||
from app.clients import AWS_CLIENT_CONFIG
|
||||
from app.clients.sms import SmsClient
|
||||
from app.cloudfoundry_config import cloud_config
|
||||
|
||||
|
||||
class AwsSnsClient(SmsClient):
|
||||
"""
|
||||
AwsSns sms client
|
||||
"""
|
||||
|
||||
def init_app(self, current_app, *args, **kwargs):
|
||||
if os.getenv("LOCALSTACK_ENDPOINT_URL"):
|
||||
self._client = client(
|
||||
"sns",
|
||||
region_name=cloud_config.sns_region,
|
||||
aws_access_key_id=cloud_config.sns_access_key,
|
||||
aws_secret_access_key=cloud_config.sns_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
endpoint_url=os.getenv("LOCALSTACK_ENDPOINT_URL"),
|
||||
)
|
||||
else:
|
||||
self._client = client(
|
||||
"sns",
|
||||
region_name=cloud_config.sns_region,
|
||||
aws_access_key_id=cloud_config.sns_access_key,
|
||||
aws_secret_access_key=cloud_config.sns_secret_key,
|
||||
config=AWS_CLIENT_CONFIG,
|
||||
)
|
||||
|
||||
super(SmsClient, self).__init__(*args, **kwargs)
|
||||
self.current_app = current_app
|
||||
self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$")
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "sns"
|
||||
|
||||
def _valid_sender_number(self, sender):
|
||||
return sender and re.match(self._valid_sender_regex, sender)
|
||||
|
||||
def send_sms(self, to, content, reference, sender=None, international=False):
|
||||
matched = False
|
||||
if "+" not in to:
|
||||
to = f"+{to}"
|
||||
|
||||
for match in phonenumbers.PhoneNumberMatcher(to, None):
|
||||
matched = True
|
||||
to = phonenumbers.format_number(
|
||||
match.number, phonenumbers.PhoneNumberFormat.E164
|
||||
)
|
||||
|
||||
# See documentation
|
||||
# https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
|
||||
attributes = {
|
||||
"AWS.SNS.SMS.SMSType": {
|
||||
"DataType": "String",
|
||||
"StringValue": "Transactional",
|
||||
}
|
||||
}
|
||||
|
||||
default_num = " ".join(self.current_app.config["AWS_US_TOLL_FREE_NUMBER"])
|
||||
if isinstance(sender, str):
|
||||
non_scrubbable = " ".join(sender)
|
||||
|
||||
self.current_app.logger.info(
|
||||
f"notify-debug-api-1385 sender {non_scrubbable} is a {type(sender)} \
|
||||
default is a {type(default_num)}"
|
||||
)
|
||||
else:
|
||||
self.current_app.logger.warning(
|
||||
f"notify-debug-api-1385 sender is type {type(sender)}!! {sender}"
|
||||
)
|
||||
if self._valid_sender_number(sender):
|
||||
self.current_app.logger.info(
|
||||
f"notify-debug-api-1385 use valid sender {non_scrubbable} instead of default {default_num}"
|
||||
)
|
||||
|
||||
attributes["AWS.MM.SMS.OriginationNumber"] = {
|
||||
"DataType": "String",
|
||||
"StringValue": sender,
|
||||
}
|
||||
else:
|
||||
self.current_app.logger.info(
|
||||
f"notify-debug-api-1385 use default {default_num} instead of invalid sender"
|
||||
)
|
||||
|
||||
attributes["AWS.MM.SMS.OriginationNumber"] = {
|
||||
"DataType": "String",
|
||||
"StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
|
||||
}
|
||||
|
||||
try:
|
||||
start_time = monotonic()
|
||||
response = self._client.publish(
|
||||
PhoneNumber=to, Message=content, MessageAttributes=attributes
|
||||
)
|
||||
except botocore.exceptions.ClientError as e:
|
||||
self.current_app.logger.exception("An error occurred sending sms")
|
||||
raise str(e)
|
||||
except Exception as e:
|
||||
self.current_app.logger.exception("An error occurred sending sms")
|
||||
raise str(e)
|
||||
finally:
|
||||
elapsed_time = monotonic() - start_time
|
||||
self.current_app.logger.info(
|
||||
"AWS SNS request finished in {}".format(elapsed_time)
|
||||
)
|
||||
return response["MessageId"]
|
||||
|
||||
if not matched:
|
||||
self.current_app.logger.error("No valid numbers found in {}".format(to))
|
||||
raise ValueError("No valid numbers found for SMS delivery")
|
||||
107
app/clients/sms/firetext.py
Normal file
107
app/clients/sms/firetext.py
Normal file
@@ -0,0 +1,107 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from time import monotonic
|
||||
from requests import request, RequestException
|
||||
|
||||
from app.clients.sms import (SmsClient, SmsClientResponseException)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Firetext will send a delivery receipt with three different status codes.
|
||||
# The `firetext_response` maps these codes to the notification statistics status and notification status.
|
||||
# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
|
||||
# the notification status to temporary-failure rather than permanent failure.
|
||||
# See the code in the notification_dao.update_notifications_status_by_id
|
||||
firetext_responses = {
|
||||
'0': 'delivered',
|
||||
'1': 'permanent-failure',
|
||||
'2': 'pending'
|
||||
}
|
||||
|
||||
|
||||
def get_firetext_responses(status):
|
||||
return firetext_responses[status]
|
||||
|
||||
|
||||
class FiretextClientResponseException(SmsClientResponseException):
|
||||
def __init__(self, response, exception):
|
||||
status_code = response.status_code if response is not None else 504
|
||||
text = response.text if response is not None else "Gateway Time-out"
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
self.exception = exception
|
||||
|
||||
def __str__(self):
|
||||
return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
|
||||
|
||||
|
||||
class FiretextClient(SmsClient):
|
||||
'''
|
||||
FireText sms client.
|
||||
'''
|
||||
|
||||
def init_app(self, current_app, statsd_client, *args, **kwargs):
|
||||
super(SmsClient, self).__init__(*args, **kwargs)
|
||||
self.current_app = current_app
|
||||
self.api_key = current_app.config.get('FIRETEXT_API_KEY')
|
||||
self.from_number = current_app.config.get('FROM_NUMBER')
|
||||
self.name = 'firetext'
|
||||
self.url = current_app.config.get('FIRETEXT_URL')
|
||||
self.statsd_client = statsd_client
|
||||
|
||||
def get_name(self):
|
||||
return self.name
|
||||
|
||||
def record_outcome(self, success, response):
|
||||
status_code = response.status_code if response else 503
|
||||
|
||||
log_message = "API {} request {} on {} response status_code {}".format(
|
||||
"POST",
|
||||
"succeeded" if success else "failed",
|
||||
self.url,
|
||||
status_code
|
||||
)
|
||||
|
||||
if success:
|
||||
self.current_app.logger.info(log_message)
|
||||
self.statsd_client.incr("clients.firetext.success")
|
||||
else:
|
||||
self.statsd_client.incr("clients.firetext.error")
|
||||
self.current_app.logger.error(log_message)
|
||||
|
||||
def send_sms(self, to, content, reference, sender=None):
|
||||
|
||||
data = {
|
||||
"apiKey": self.api_key,
|
||||
"from": self.from_number if sender is None else sender,
|
||||
"to": to.replace('+', ''),
|
||||
"message": content,
|
||||
"reference": reference
|
||||
}
|
||||
|
||||
start_time = monotonic()
|
||||
try:
|
||||
response = request(
|
||||
"POST",
|
||||
self.url,
|
||||
data=data,
|
||||
timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
try:
|
||||
json.loads(response.text)
|
||||
if response.json()['code'] != 0:
|
||||
raise ValueError()
|
||||
except (ValueError, AttributeError) as e:
|
||||
self.record_outcome(False, response)
|
||||
raise FiretextClientResponseException(response=response, exception=e)
|
||||
self.record_outcome(True, response)
|
||||
except RequestException as e:
|
||||
self.record_outcome(False, e.response)
|
||||
raise FiretextClientResponseException(response=e.response, exception=e)
|
||||
finally:
|
||||
elapsed_time = monotonic() - start_time
|
||||
self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
|
||||
self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
|
||||
return response
|
||||
102
app/clients/sms/mmg.py
Normal file
102
app/clients/sms/mmg.py
Normal file
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
from time import monotonic
|
||||
from requests import (request, RequestException)
|
||||
from app.clients.sms import (SmsClient, SmsClientResponseException)
|
||||
|
||||
mmg_response_map = {
|
||||
'2': 'permanent-failure',
|
||||
'3': 'delivered',
|
||||
'4': 'temporary-failure',
|
||||
'5': 'permanent-failure'
|
||||
}
|
||||
|
||||
|
||||
def get_mmg_responses(status):
|
||||
return mmg_response_map[status]
|
||||
|
||||
|
||||
class MMGClientResponseException(SmsClientResponseException):
|
||||
def __init__(self, response, exception):
|
||||
status_code = response.status_code if response is not None else 504
|
||||
text = response.text if response is not None else "Gateway Time-out"
|
||||
|
||||
self.status_code = status_code
|
||||
self.text = text
|
||||
self.exception = exception
|
||||
|
||||
def __str__(self):
|
||||
return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
|
||||
|
||||
|
||||
class MMGClient(SmsClient):
|
||||
'''
|
||||
MMG sms client
|
||||
'''
|
||||
|
||||
def init_app(self, current_app, statsd_client, *args, **kwargs):
|
||||
super(SmsClient, self).__init__(*args, **kwargs)
|
||||
self.current_app = current_app
|
||||
self.api_key = current_app.config.get('MMG_API_KEY')
|
||||
self.from_number = current_app.config.get('FROM_NUMBER')
|
||||
self.name = 'mmg'
|
||||
self.statsd_client = statsd_client
|
||||
self.mmg_url = current_app.config.get('MMG_URL')
|
||||
|
||||
def record_outcome(self, success, response):
|
||||
status_code = response.status_code if response else 503
|
||||
log_message = "API {} request {} on {} response status_code {}".format(
|
||||
"POST",
|
||||
"succeeded" if success else "failed",
|
||||
self.mmg_url,
|
||||
status_code
|
||||
)
|
||||
|
||||
if success:
|
||||
self.current_app.logger.info(log_message)
|
||||
self.statsd_client.incr("clients.mmg.success")
|
||||
else:
|
||||
self.statsd_client.incr("clients.mmg.error")
|
||||
self.current_app.logger.error(log_message)
|
||||
|
||||
def get_name(self):
|
||||
return self.name
|
||||
|
||||
def send_sms(self, to, content, reference, multi=True, sender=None):
|
||||
data = {
|
||||
"reqType": "BULK",
|
||||
"MSISDN": to,
|
||||
"msg": content,
|
||||
"sender": self.from_number if sender is None else sender,
|
||||
"cid": reference,
|
||||
"multi": multi
|
||||
}
|
||||
|
||||
start_time = monotonic()
|
||||
try:
|
||||
response = request(
|
||||
"POST",
|
||||
self.mmg_url,
|
||||
data=json.dumps(data),
|
||||
headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Basic {}'.format(self.api_key)
|
||||
},
|
||||
timeout=60
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
try:
|
||||
json.loads(response.text)
|
||||
except (ValueError, AttributeError) as e:
|
||||
self.record_outcome(False, response)
|
||||
raise MMGClientResponseException(response=response, exception=e)
|
||||
self.record_outcome(True, response)
|
||||
except RequestException as e:
|
||||
self.record_outcome(False, e.response)
|
||||
raise MMGClientResponseException(response=e.response, exception=e)
|
||||
finally:
|
||||
elapsed_time = monotonic() - start_time
|
||||
self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
|
||||
self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
|
||||
|
||||
return response
|
||||
@@ -1,110 +1,21 @@
|
||||
"""
|
||||
Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
|
||||
on local/aws boxes
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
from os import getenv
|
||||
|
||||
|
||||
class CloudfoundryConfig:
|
||||
def __init__(self):
|
||||
self.parsed_services = json.loads(getenv("VCAP_SERVICES") or "{}")
|
||||
buckets = self.parsed_services.get("s3") or []
|
||||
self.s3_buckets = {bucket["name"]: bucket["credentials"] for bucket in buckets}
|
||||
self._empty_bucket_credentials = {
|
||||
"bucket": "",
|
||||
"access_key_id": "",
|
||||
"secret_access_key": "",
|
||||
"region": "",
|
||||
}
|
||||
|
||||
@property
|
||||
def database_url(self):
|
||||
return getenv("DATABASE_URL", "").replace("postgres://", "postgresql://")
|
||||
|
||||
@property
|
||||
def redis_url(self):
|
||||
try:
|
||||
return self.parsed_services["aws-elasticache-redis"][0]["credentials"][
|
||||
"uri"
|
||||
].replace("redis://", "rediss://")
|
||||
except KeyError:
|
||||
return getenv("REDIS_URL")
|
||||
|
||||
def s3_credentials(self, service_name):
|
||||
return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
|
||||
|
||||
@property
|
||||
def ses_email_domain(self):
|
||||
try:
|
||||
domain_arn = self._ses_credentials("domain_arn")
|
||||
except KeyError:
|
||||
domain_arn = getenv("SES_DOMAIN_ARN", "dev.notify.gov")
|
||||
return domain_arn.split("/")[-1]
|
||||
|
||||
# TODO remove this after notifications-api #258
|
||||
@property
|
||||
def ses_domain_arn(self):
|
||||
try:
|
||||
domain_arn = self._ses_credentials("domain_arn")
|
||||
except KeyError:
|
||||
domain_arn = getenv("SES_DOMAIN_ARN", "dev.notify.gov")
|
||||
return domain_arn
|
||||
|
||||
@property
|
||||
def ses_region(self):
|
||||
try:
|
||||
return self._ses_credentials("region")
|
||||
except KeyError:
|
||||
return getenv("SES_AWS_REGION", "us-west-1")
|
||||
|
||||
@property
|
||||
def ses_access_key(self):
|
||||
try:
|
||||
return self._ses_credentials("smtp_user")
|
||||
except KeyError:
|
||||
return getenv("SES_AWS_ACCESS_KEY_ID")
|
||||
|
||||
@property
|
||||
def ses_secret_key(self):
|
||||
try:
|
||||
return self._ses_credentials("secret_access_key")
|
||||
except KeyError:
|
||||
return getenv("SES_AWS_SECRET_ACCESS_KEY")
|
||||
|
||||
@property
|
||||
def sns_access_key(self):
|
||||
try:
|
||||
return self._sns_credentials("aws_access_key_id")
|
||||
except KeyError:
|
||||
return getenv("SNS_AWS_ACCESS_KEY_ID")
|
||||
|
||||
@property
|
||||
def sns_secret_key(self):
|
||||
try:
|
||||
return self._sns_credentials("aws_secret_access_key")
|
||||
except KeyError:
|
||||
return getenv("SNS_AWS_SECRET_ACCESS_KEY")
|
||||
|
||||
@property
|
||||
def sns_region(self):
|
||||
try:
|
||||
return self._sns_credentials("region")
|
||||
except KeyError:
|
||||
return getenv("SNS_AWS_REGION", "us-west-1")
|
||||
|
||||
@property
|
||||
def sns_topic_arns(self):
|
||||
try:
|
||||
return [
|
||||
self._ses_credentials("bounce_topic_arn"),
|
||||
self._ses_credentials("complaint_topic_arn"),
|
||||
self._ses_credentials("delivery_topic_arn"),
|
||||
]
|
||||
except KeyError:
|
||||
return []
|
||||
|
||||
def _ses_credentials(self, key):
|
||||
return self.parsed_services["datagov-smtp"][0]["credentials"][key]
|
||||
|
||||
def _sns_credentials(self, key):
|
||||
return self.parsed_services["ttsnotify-sms"][0]["credentials"][key]
|
||||
def extract_cloudfoundry_config():
|
||||
vcap_services = json.loads(os.environ['VCAP_SERVICES'])
|
||||
set_config_env_vars(vcap_services)
|
||||
|
||||
|
||||
cloud_config = CloudfoundryConfig()
|
||||
def set_config_env_vars(vcap_services):
|
||||
# Postgres config
|
||||
os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']
|
||||
|
||||
vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
|
||||
os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
|
||||
os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
|
||||
|
||||
1699
app/commands.py
1699
app/commands.py
File diff suppressed because it is too large
Load Diff
@@ -6,43 +6,38 @@ from app.complaint.complaint_schema import complaint_count_request
|
||||
from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
|
||||
from app.errors import register_errors
|
||||
from app.schema_validation import validate
|
||||
from app.utils import pagination_links, utc_now
|
||||
from app.utils import pagination_links
|
||||
|
||||
complaint_blueprint = Blueprint("complaint", __name__, url_prefix="/complaint")
|
||||
complaint_blueprint = Blueprint('complaint', __name__, url_prefix='/complaint')
|
||||
|
||||
register_errors(complaint_blueprint)
|
||||
|
||||
|
||||
@complaint_blueprint.route("", methods=["GET"])
|
||||
@complaint_blueprint.route('', methods=['GET'])
|
||||
def get_all_complaints():
|
||||
page = int(request.args.get("page", 1))
|
||||
page = int(request.args.get('page', 1))
|
||||
pagination = fetch_paginated_complaints(page=page)
|
||||
|
||||
return (
|
||||
jsonify(
|
||||
complaints=[x.serialize() for x in pagination.items],
|
||||
links=pagination_links(
|
||||
pagination, ".get_all_complaints", **request.args.to_dict()
|
||||
),
|
||||
),
|
||||
200,
|
||||
)
|
||||
return jsonify(
|
||||
complaints=[x.serialize() for x in pagination.items],
|
||||
links=pagination_links(
|
||||
pagination,
|
||||
'.get_all_complaints',
|
||||
**request.args.to_dict()
|
||||
)
|
||||
), 200
|
||||
|
||||
|
||||
@complaint_blueprint.route("/count-by-date-range", methods=["GET"])
|
||||
@complaint_blueprint.route('/count-by-date-range', methods=['GET'])
|
||||
def get_complaint_count():
|
||||
if request.args:
|
||||
validate(request.args, complaint_count_request)
|
||||
|
||||
# If start and end date are not set, we are expecting today's stats.
|
||||
today = str(utc_now().date())
|
||||
today = str(datetime.utcnow().date())
|
||||
|
||||
start_date = datetime.strptime(
|
||||
request.args.get("start_date", today), "%Y-%m-%d"
|
||||
).date()
|
||||
end_date = datetime.strptime(request.args.get("end_date", today), "%Y-%m-%d").date()
|
||||
count_of_complaints = fetch_count_of_complaints(
|
||||
start_date=start_date, end_date=end_date
|
||||
)
|
||||
start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
|
||||
end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()
|
||||
count_of_complaints = fetch_count_of_complaints(start_date=start_date, end_date=end_date)
|
||||
|
||||
return jsonify(count_of_complaints), 200
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
|
||||
complaint_count_request = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"description": "complaint count request schema",
|
||||
"type": "object",
|
||||
"title": "Complaint count request",
|
||||
"properties": {
|
||||
"start_date": {"type": ["string", "null"], "format": "date"},
|
||||
"end_date": {"type": ["string", "null"], "format": "date"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
778
app/config.py
778
app/config.py
@@ -1,427 +1,525 @@
|
||||
from datetime import timedelta
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from os import getenv, path
|
||||
|
||||
from boto3 import Session
|
||||
from celery.schedules import crontab
|
||||
from kombu import Exchange, Queue
|
||||
|
||||
import notifications_utils
|
||||
from app.clients import AWS_CLIENT_CONFIG
|
||||
from app.cloudfoundry_config import cloud_config
|
||||
if os.environ.get('VCAP_SERVICES'):
|
||||
# on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
|
||||
# standard environment variables from it
|
||||
from app.cloudfoundry_config import extract_cloudfoundry_config
|
||||
|
||||
extract_cloudfoundry_config()
|
||||
|
||||
|
||||
class QueueNames(object):
|
||||
PERIODIC = "periodic-tasks"
|
||||
DATABASE = "database-tasks"
|
||||
SEND_SMS = "send-sms-tasks"
|
||||
CHECK_SMS = "check-sms_tasks"
|
||||
SEND_EMAIL = "send-email-tasks"
|
||||
REPORTING = "reporting-tasks"
|
||||
JOBS = "job-tasks"
|
||||
RETRY = "retry-tasks"
|
||||
NOTIFY = "notify-internal-tasks"
|
||||
CALLBACKS = "service-callbacks"
|
||||
CALLBACKS_RETRY = "service-callbacks-retry"
|
||||
SMS_CALLBACKS = "sms-callbacks"
|
||||
ANTIVIRUS = "antivirus-tasks"
|
||||
SAVE_API_EMAIL = "save-api-email-tasks"
|
||||
SAVE_API_SMS = "save-api-sms-tasks"
|
||||
PERIODIC = 'periodic-tasks'
|
||||
PRIORITY = 'priority-tasks'
|
||||
DATABASE = 'database-tasks'
|
||||
SEND_SMS = 'send-sms-tasks'
|
||||
SEND_EMAIL = 'send-email-tasks'
|
||||
RESEARCH_MODE = 'research-mode-tasks'
|
||||
REPORTING = 'reporting-tasks'
|
||||
JOBS = 'job-tasks'
|
||||
RETRY = 'retry-tasks'
|
||||
NOTIFY = 'notify-internal-tasks'
|
||||
PROCESS_FTP = 'process-ftp-tasks'
|
||||
CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
|
||||
CALLBACKS = 'service-callbacks'
|
||||
LETTERS = 'letter-tasks'
|
||||
ANTIVIRUS = 'antivirus-tasks'
|
||||
SANITISE_LETTERS = 'sanitise-letter-tasks'
|
||||
|
||||
@staticmethod
|
||||
def all_queues():
|
||||
return [
|
||||
QueueNames.PRIORITY,
|
||||
QueueNames.PERIODIC,
|
||||
QueueNames.DATABASE,
|
||||
QueueNames.SEND_SMS,
|
||||
QueueNames.CHECK_SMS,
|
||||
QueueNames.SEND_EMAIL,
|
||||
QueueNames.RESEARCH_MODE,
|
||||
QueueNames.REPORTING,
|
||||
QueueNames.JOBS,
|
||||
QueueNames.RETRY,
|
||||
QueueNames.NOTIFY,
|
||||
QueueNames.CREATE_LETTERS_PDF,
|
||||
QueueNames.CALLBACKS,
|
||||
QueueNames.CALLBACKS_RETRY,
|
||||
QueueNames.SMS_CALLBACKS,
|
||||
QueueNames.SAVE_API_EMAIL,
|
||||
QueueNames.SAVE_API_SMS,
|
||||
QueueNames.LETTERS,
|
||||
]
|
||||
|
||||
|
||||
class TaskNames(object):
|
||||
PROCESS_INCOMPLETE_JOBS = "process-incomplete-jobs"
|
||||
SCAN_FILE = "scan-file"
|
||||
|
||||
|
||||
session = Session(
|
||||
aws_access_key_id=getenv("CSV_AWS_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=getenv("CSV_AWS_SECRET_ACCESS_KEY"),
|
||||
region_name=getenv("CSV_AWS_REGION"),
|
||||
)
|
||||
PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
|
||||
ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
|
||||
SCAN_FILE = 'scan-file'
|
||||
SANITISE_LETTER = 'sanitise-and-upload-letter'
|
||||
|
||||
|
||||
class Config(object):
|
||||
NOTIFY_APP_NAME = "api"
|
||||
DEFAULT_REDIS_EXPIRE_TIME = 4 * 24 * 60 * 60
|
||||
NOTIFY_ENVIRONMENT = getenv("NOTIFY_ENVIRONMENT", "development")
|
||||
# URL of admin app
|
||||
ADMIN_BASE_URL = getenv("ADMIN_BASE_URL", "http://localhost:6012")
|
||||
ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
|
||||
|
||||
# URL of api app (on AWS this is the internal api endpoint)
|
||||
API_HOST_NAME = getenv("API_HOST_NAME", "http://localhost:6011")
|
||||
API_HOST_NAME = os.getenv('API_HOST_NAME')
|
||||
|
||||
# admin app api key
|
||||
ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET')
|
||||
|
||||
# Credentials
|
||||
# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
|
||||
# ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
|
||||
ADMIN_CLIENT_ID = getenv("ADMIN_CLIENT_ID", "notify-admin")
|
||||
INTERNAL_CLIENT_API_KEYS = json.loads(
|
||||
getenv(
|
||||
"INTERNAL_CLIENT_API_KEYS",
|
||||
('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv("ADMIN_CLIENT_SECRET"))),
|
||||
)
|
||||
)
|
||||
ALLOW_EXPIRED_API_TOKEN = False
|
||||
# encyption secret/salt
|
||||
SECRET_KEY = getenv("SECRET_KEY")
|
||||
DANGEROUS_SALT = getenv("DANGEROUS_SALT")
|
||||
ROUTE_SECRET_KEY_1 = getenv("ROUTE_SECRET_KEY_1", "dev-route-secret-key-1")
|
||||
ROUTE_SECRET_KEY_2 = getenv("ROUTE_SECRET_KEY_2", "dev-route-secret-key-2")
|
||||
SECRET_KEY = os.getenv('SECRET_KEY')
|
||||
DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
|
||||
|
||||
# DB settings
|
||||
SQLALCHEMY_DATABASE_URI = cloud_config.database_url
|
||||
SQLALCHEMY_RECORD_QUERIES = False
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 40))
|
||||
SQLALCHEMY_POOL_TIMEOUT = 30
|
||||
SQLALCHEMY_POOL_RECYCLE = 300
|
||||
SQLALCHEMY_STATEMENT_TIMEOUT = 1200
|
||||
PAGE_SIZE = 20
|
||||
API_PAGE_SIZE = 250
|
||||
REDIS_URL = cloud_config.redis_url
|
||||
REDIS_ENABLED = getenv("REDIS_ENABLED", "1") == "1"
|
||||
# DB conection string
|
||||
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
|
||||
|
||||
# MMG API Key
|
||||
MMG_API_KEY = os.getenv('MMG_API_KEY')
|
||||
|
||||
# Firetext API Key
|
||||
FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
|
||||
|
||||
# Hosted graphite statsd prefix
|
||||
STATSD_PREFIX = os.getenv('STATSD_PREFIX')
|
||||
|
||||
# Prefix to identify queues in SQS
|
||||
NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
|
||||
|
||||
# URL of redis instance
|
||||
REDIS_URL = os.getenv('REDIS_URL')
|
||||
REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
|
||||
EXPIRE_CACHE_TEN_MINUTES = 600
|
||||
EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
|
||||
|
||||
# AWS Settings
|
||||
AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER")
|
||||
# Whether to ignore POSTs from SNS for replies to SMS we sent
|
||||
RECEIVE_INBOUND_SMS = False
|
||||
NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
|
||||
SES_STUB_URL = (
|
||||
None # TODO: set to a URL in env and remove this to use a stubbed SES service
|
||||
)
|
||||
# AWS SNS topics for delivery receipts
|
||||
VALIDATE_SNS_TOPICS = True
|
||||
VALID_SNS_TOPICS = cloud_config.sns_topic_arns
|
||||
|
||||
# these should always add up to 100%
|
||||
SMS_PROVIDER_RESTING_POINTS = {
|
||||
"sns": 100,
|
||||
}
|
||||
# Performance platform
|
||||
PERFORMANCE_PLATFORM_ENABLED = False
|
||||
PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
|
||||
|
||||
# Zendesk
|
||||
ZENDESK_API_KEY = getenv("ZENDESK_API_KEY")
|
||||
ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')
|
||||
|
||||
# Logging
|
||||
DEBUG = False
|
||||
NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')
|
||||
|
||||
# Cronitor
|
||||
CRONITOR_ENABLED = False
|
||||
CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}'))
|
||||
|
||||
# Antivirus
|
||||
ANTIVIRUS_ENABLED = getenv("ANTIVIRUS_ENABLED", "1") == "1"
|
||||
ANTIVIRUS_ENABLED = True
|
||||
|
||||
SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days
|
||||
###########################
|
||||
# Default config values ###
|
||||
###########################
|
||||
|
||||
NOTIFY_ENVIRONMENT = 'development'
|
||||
ADMIN_CLIENT_USER_NAME = 'notify-admin'
|
||||
AWS_REGION = 'eu-west-1'
|
||||
INVITATION_EXPIRATION_DAYS = 2
|
||||
TEST_MESSAGE_FILENAME = "Test message"
|
||||
ONE_OFF_MESSAGE_FILENAME = "Report"
|
||||
MAX_VERIFY_CODE_COUNT = 5
|
||||
MAX_FAILED_LOGIN_COUNT = 10
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
NOTIFY_APP_NAME = 'api'
|
||||
SQLALCHEMY_RECORD_QUERIES = False
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5))
|
||||
SQLALCHEMY_POOL_TIMEOUT = 30
|
||||
SQLALCHEMY_POOL_RECYCLE = 300
|
||||
SQLALCHEMY_STATEMENT_TIMEOUT = 1200
|
||||
PAGE_SIZE = 50
|
||||
API_PAGE_SIZE = 250
|
||||
TEST_MESSAGE_FILENAME = 'Test message'
|
||||
ONE_OFF_MESSAGE_FILENAME = 'Report'
|
||||
MAX_VERIFY_CODE_COUNT = 10
|
||||
|
||||
# Default data
|
||||
CONFIG_FILES = path.dirname(__file__) + "/config_files/"
|
||||
# be careful increasing this size without being sure that we won't see slowness in pysftp
|
||||
MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb
|
||||
MAX_LETTER_PDF_COUNT_PER_ZIP = 500
|
||||
|
||||
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
|
||||
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
|
||||
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
|
||||
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
|
||||
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
|
||||
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
|
||||
PASSWORD_RESET_TEMPLATE_ID = (
|
||||
"474e9242-823b-4f99-813d-ed392e7f1201" # nosec B105 - this is not a password
|
||||
)
|
||||
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb"
|
||||
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884"
|
||||
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
|
||||
ORGANIZATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
|
||||
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
|
||||
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"
|
||||
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = (
|
||||
"a42f1d17-9404-46d5-a647-d013bdfca3e1"
|
||||
)
|
||||
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
|
||||
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = (
|
||||
"c20206d5-bf03-4002-9a90-37d5032d9e84"
|
||||
)
|
||||
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = (
|
||||
"522b6657-5ca5-4368-a294-6b527703bd0b"
|
||||
)
|
||||
NOTIFY_INTERNATIONAL_SMS_SENDER = getenv("AWS_US_TOLL_FREE_NUMBER")
|
||||
LETTERS_VOLUME_EMAIL_TEMPLATE_ID = "11fad854-fd38-4a7c-bd17-805fb13dfc12"
|
||||
NHS_EMAIL_BRANDING_ID = "a7dc4e56-660b-4db7-8cff-12c37b12b5ea"
|
||||
# we only need real email in Live environment (production)
|
||||
DVLA_EMAIL_ADDRESSES = json.loads(getenv("DVLA_EMAIL_ADDRESSES", "[]"))
|
||||
CHECK_PROXY_HEADER = False
|
||||
|
||||
current_minute = (datetime.now().minute + 1) % 60
|
||||
|
||||
S3_CLIENT = session.client("s3")
|
||||
S3_RESOURCE = session.resource("s3", config=AWS_CLIENT_CONFIG)
|
||||
|
||||
CELERY = {
|
||||
"broker_connection_retry_on_startup": True,
|
||||
"worker_max_tasks_per_child": 500,
|
||||
"task_ignore_result": True,
|
||||
"result_persistent": False,
|
||||
"broker_url": REDIS_URL,
|
||||
"broker_transport_options": {
|
||||
"visibility_timeout": 310,
|
||||
},
|
||||
"timezone": getenv("TIMEZONE", "UTC"),
|
||||
"imports": [
|
||||
"app.celery.tasks",
|
||||
"app.celery.scheduled_tasks",
|
||||
"app.celery.reporting_tasks",
|
||||
"app.celery.nightly_tasks",
|
||||
],
|
||||
# this is overriden by the -Q command, but locally, we should read from all queues
|
||||
"task_queues": [
|
||||
Queue(queue, Exchange("default"), routing_key=queue)
|
||||
for queue in QueueNames.all_queues()
|
||||
],
|
||||
"beat_schedule": {
|
||||
# app/celery/scheduled_tasks.py
|
||||
"run-scheduled-jobs": {
|
||||
"task": "run-scheduled-jobs",
|
||||
"schedule": crontab(minute="0,15,30,45"),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"delete-verify-codes": {
|
||||
"task": "delete-verify-codes",
|
||||
"schedule": timedelta(minutes=63),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"process-delivery-receipts": {
|
||||
"task": "process-delivery-receipts",
|
||||
"schedule": timedelta(minutes=2),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"cleanup-delivery-receipts": {
|
||||
"task": "cleanup-delivery-receipts",
|
||||
"schedule": timedelta(minutes=82),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"batch-insert-notifications": {
|
||||
"task": "batch-insert-notifications",
|
||||
"schedule": 10.0,
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"expire-or-delete-invitations": {
|
||||
"task": "expire-or-delete-invitations",
|
||||
"schedule": timedelta(minutes=66),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"check-job-status": {
|
||||
"task": "check-job-status",
|
||||
"schedule": crontab(),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"check-for-missing-rows-in-completed-jobs": {
|
||||
"task": "check-for-missing-rows-in-completed-jobs",
|
||||
"schedule": crontab(minute="*/10"),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"replay-created-notifications": {
|
||||
"task": "replay-created-notifications",
|
||||
"schedule": crontab(minute="0, 15, 30, 45"),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
# app/celery/nightly_tasks.py
|
||||
"timeout-sending-notifications": {
|
||||
"task": "timeout-sending-notifications",
|
||||
"schedule": crontab(hour=4, minute=5),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"create-nightly-billing": {
|
||||
"task": "create-nightly-billing",
|
||||
"schedule": crontab(hour=4, minute=15),
|
||||
"options": {"queue": QueueNames.REPORTING},
|
||||
},
|
||||
"create-nightly-notification-status": {
|
||||
"task": "create-nightly-notification-status",
|
||||
"schedule": crontab(
|
||||
hour=4, minute=30
|
||||
), # after 'timeout-sending-notifications'
|
||||
"options": {"queue": QueueNames.REPORTING},
|
||||
},
|
||||
"delete-notifications-older-than-retention": {
|
||||
"task": "delete-notifications-older-than-retention",
|
||||
"schedule": crontab(
|
||||
hour=7, minute=0
|
||||
), # after 'create-nightly-notification-status'
|
||||
"options": {"queue": QueueNames.REPORTING},
|
||||
},
|
||||
"delete-inbound-sms": {
|
||||
"task": "delete-inbound-sms",
|
||||
"schedule": crontab(hour=5, minute=40),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"save-daily-notification-processing-time": {
|
||||
"task": "save-daily-notification-processing-time",
|
||||
"schedule": crontab(hour=6, minute=0),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"delete_old_s3_objects": {
|
||||
"task": "delete-old-s3-objects",
|
||||
"schedule": crontab(hour=7, minute=10),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"regenerate-job-cache": {
|
||||
"task": "regenerate-job-cache",
|
||||
"schedule": crontab(minute="*/30"),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"regenerate-job-cache-on-startup": {
|
||||
"task": "regenerate-job-cache",
|
||||
"schedule": crontab(
|
||||
minute=current_minute
|
||||
), # Runs once at the next minute
|
||||
"options": {
|
||||
"queue": QueueNames.PERIODIC,
|
||||
"expires": 60,
|
||||
}, # Ensure it doesn't run if missed
|
||||
},
|
||||
"clean-job-cache": {
|
||||
"task": "clean-job-cache",
|
||||
"schedule": crontab(hour=2, minute=11),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"cleanup-unfinished-jobs": {
|
||||
"task": "cleanup-unfinished-jobs",
|
||||
"schedule": crontab(hour=4, minute=5),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"remove-sms-email-jobs": {
|
||||
"task": "remove-sms-email-jobs",
|
||||
"schedule": crontab(hour=8, minute=0),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
"check-for-services-with-high-failure-rates-or-sending-to-tv-numbers": {
|
||||
"task": "check-for-services-with-high-failure-rates-or-sending-to-tv-numbers",
|
||||
"schedule": crontab(day_of_week="mon-fri", hour=14, minute=30),
|
||||
"options": {"queue": QueueNames.PERIODIC},
|
||||
},
|
||||
},
|
||||
# these should always add up to 100%
|
||||
SMS_PROVIDER_RESTING_POINTS = {
|
||||
'mmg': 60,
|
||||
'firetext': 40
|
||||
}
|
||||
|
||||
# we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
|
||||
if getenv("CELERYD_PREFETCH_MULTIPLIER"):
|
||||
CELERY["worker_prefetch_multiplier"] = getenv("CELERYD_PREFETCH_MULTIPLIER")
|
||||
NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
|
||||
NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
|
||||
INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
|
||||
SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
|
||||
EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
|
||||
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
|
||||
PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
|
||||
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
|
||||
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
|
||||
SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
|
||||
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573'
|
||||
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = 'c73f1d71-4049-46d5-a647-d013bdeca3f0'
|
||||
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = '8a31520f-4751-4789-8ea1-fe54496725eb'
|
||||
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = 'a42f1d17-9404-46d5-a647-d013bdfca3e1'
|
||||
MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
|
||||
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
|
||||
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
|
||||
MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'
|
||||
|
||||
FROM_NUMBER = "development"
|
||||
BROKER_URL = 'sqs://'
|
||||
BROKER_TRANSPORT_OPTIONS = {
|
||||
'region': AWS_REGION,
|
||||
'polling_interval': 1, # 1 second
|
||||
'visibility_timeout': 310,
|
||||
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
|
||||
}
|
||||
CELERY_ENABLE_UTC = True
|
||||
CELERY_TIMEZONE = 'Europe/London'
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_IMPORTS = (
|
||||
'app.celery.tasks',
|
||||
'app.celery.scheduled_tasks',
|
||||
'app.celery.reporting_tasks',
|
||||
'app.celery.nightly_tasks',
|
||||
)
|
||||
CELERYBEAT_SCHEDULE = {
|
||||
# app/celery/scheduled_tasks.py
|
||||
'run-scheduled-jobs': {
|
||||
'task': 'run-scheduled-jobs',
|
||||
'schedule': crontab(minute=1),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-verify-codes': {
|
||||
'task': 'delete-verify-codes',
|
||||
'schedule': timedelta(minutes=63),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-invitations': {
|
||||
'task': 'delete-invitations',
|
||||
'schedule': timedelta(minutes=66),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'switch-current-sms-provider-on-slow-delivery': {
|
||||
'task': 'switch-current-sms-provider-on-slow-delivery',
|
||||
'schedule': crontab(), # Every minute
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'check-job-status': {
|
||||
'task': 'check-job-status',
|
||||
'schedule': crontab(),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'tend-providers-back-to-middle': {
|
||||
'task': 'tend-providers-back-to-middle',
|
||||
'schedule': crontab(minute='*/5'),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'check-for-missing-rows-in-completed-jobs': {
|
||||
'task': 'check-for-missing-rows-in-completed-jobs',
|
||||
'schedule': crontab(minute='*/10'),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'replay-created-notifications': {
|
||||
'task': 'replay-created-notifications',
|
||||
'schedule': crontab(minute='0, 15, 30, 45'),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
# app/celery/nightly_tasks.py
|
||||
'timeout-sending-notifications': {
|
||||
'task': 'timeout-sending-notifications',
|
||||
'schedule': crontab(hour=0, minute=5),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'create-nightly-billing': {
|
||||
'task': 'create-nightly-billing',
|
||||
'schedule': crontab(hour=0, minute=15),
|
||||
'options': {'queue': QueueNames.REPORTING}
|
||||
},
|
||||
'create-nightly-notification-status': {
|
||||
'task': 'create-nightly-notification-status',
|
||||
'schedule': crontab(hour=0, minute=30), # after 'timeout-sending-notifications'
|
||||
'options': {'queue': QueueNames.REPORTING}
|
||||
},
|
||||
'delete-sms-notifications': {
|
||||
'task': 'delete-sms-notifications',
|
||||
'schedule': crontab(hour=4, minute=15), # after 'create-nightly-notification-status'
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-email-notifications': {
|
||||
'task': 'delete-email-notifications',
|
||||
'schedule': crontab(hour=4, minute=30), # after 'create-nightly-notification-status'
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-letter-notifications': {
|
||||
'task': 'delete-letter-notifications',
|
||||
'schedule': crontab(hour=4, minute=45), # after 'create-nightly-notification-status'
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-inbound-sms': {
|
||||
'task': 'delete-inbound-sms',
|
||||
'schedule': crontab(hour=1, minute=40),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
|
||||
'send-daily-performance-platform-stats': {
|
||||
'task': 'send-daily-performance-platform-stats',
|
||||
'schedule': crontab(hour=2, minute=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'remove_transformed_dvla_files': {
|
||||
'task': 'remove_transformed_dvla_files',
|
||||
'schedule': crontab(hour=3, minute=40),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'remove_sms_email_jobs': {
|
||||
'task': 'remove_sms_email_jobs',
|
||||
'schedule': crontab(hour=4, minute=0),
|
||||
'options': {'queue': QueueNames.PERIODIC},
|
||||
},
|
||||
'remove_letter_jobs': {
|
||||
'task': 'remove_letter_jobs',
|
||||
'schedule': crontab(hour=4, minute=20), # this has to run AFTER remove_transformed_dvla_files
|
||||
# since we mark jobs as archived
|
||||
'options': {'queue': QueueNames.PERIODIC},
|
||||
},
|
||||
'check-templated-letter-state': {
|
||||
'task': 'check-templated-letter-state',
|
||||
'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'check-precompiled-letter-state': {
|
||||
'task': 'check-precompiled-letter-state',
|
||||
'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
|
||||
'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
|
||||
'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'raise-alert-if-letter-notifications-still-sending': {
|
||||
'task': 'raise-alert-if-letter-notifications-still-sending',
|
||||
'schedule': crontab(hour=15, minute=30),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
# The collate-letter-pdf does assume it is called in an hour that BST does not make a
|
||||
# difference to the truncate date which translates to the filename to process
|
||||
'collate-letter-pdfs-for-day': {
|
||||
'task': 'collate-letter-pdfs-for-day',
|
||||
'schedule': crontab(hour=17, minute=50),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'raise-alert-if-no-letter-ack-file': {
|
||||
'task': 'raise-alert-if-no-letter-ack-file',
|
||||
'schedule': crontab(hour=23, minute=00),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
}
|
||||
CELERY_QUEUES = []
|
||||
|
||||
FROM_NUMBER = 'development'
|
||||
|
||||
STATSD_HOST = os.getenv('STATSD_HOST')
|
||||
STATSD_PORT = 8125
|
||||
STATSD_ENABLED = bool(STATSD_HOST)
|
||||
|
||||
SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days
|
||||
|
||||
SIMULATED_EMAIL_ADDRESSES = (
|
||||
"simulate-delivered@notifications.service.gov.uk",
|
||||
"simulate-delivered-2@notifications.service.gov.uk",
|
||||
"simulate-delivered-3@notifications.service.gov.uk",
|
||||
'simulate-delivered@notifications.service.gov.uk',
|
||||
'simulate-delivered-2@notifications.service.gov.uk',
|
||||
'simulate-delivered-3@notifications.service.gov.uk',
|
||||
)
|
||||
# 7755 is success, 7167 is failure
|
||||
SIMULATED_SMS_NUMBERS = ("+14254147755", "+14254147167")
|
||||
|
||||
SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
|
||||
|
||||
DVLA_BUCKETS = {
|
||||
'job': '{}-dvla-file-per-job'.format(os.getenv('NOTIFY_ENVIRONMENT')),
|
||||
'notification': '{}-dvla-letter-api-files'.format(os.getenv('NOTIFY_ENVIRONMENT'))
|
||||
}
|
||||
|
||||
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
|
||||
|
||||
TOTAL_MESSAGE_LIMIT = 5000000
|
||||
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
|
||||
FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
|
||||
MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
|
||||
MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
|
||||
|
||||
DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT
|
||||
ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
|
||||
ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
|
||||
|
||||
HIGH_VOLUME_SERVICE = json.loads(getenv("HIGH_VOLUME_SERVICE", "[]"))
|
||||
# Format is as follows:
|
||||
# {"dataset_1": "token_1", ...}
|
||||
PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
|
||||
|
||||
DOCUMENT_DOWNLOAD_API_HOST = getenv(
|
||||
"DOCUMENT_DOWNLOAD_API_HOST", "http://localhost:7000"
|
||||
)
|
||||
DOCUMENT_DOWNLOAD_API_KEY = getenv("DOCUMENT_DOWNLOAD_API_KEY", "auth-token")
|
||||
TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
|
||||
TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
|
||||
|
||||
DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000')
|
||||
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
|
||||
|
||||
# these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
|
||||
MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
|
||||
FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
|
||||
|
||||
AWS_REGION = 'eu-west-1'
|
||||
|
||||
|
||||
def _s3_credentials_from_env(bucket_prefix):
|
||||
return {
|
||||
"bucket": getenv(f"{bucket_prefix}_BUCKET_NAME"),
|
||||
"access_key_id": getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"),
|
||||
"secret_access_key": getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"),
|
||||
"region": getenv(f"{bucket_prefix}_AWS_REGION"),
|
||||
}
|
||||
|
||||
######################
|
||||
# Config overrides ###
|
||||
######################
|
||||
|
||||
class Development(Config):
|
||||
DEBUG = True
|
||||
NOTIFY_LOG_LEVEL = "DEBUG"
|
||||
SQLALCHEMY_ECHO = False
|
||||
DVLA_EMAIL_ADDRESSES = ["success@simulator.amazonses.com"]
|
||||
|
||||
# Buckets
|
||||
CSV_UPLOAD_BUCKET = _s3_credentials_from_env("CSV")
|
||||
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
|
||||
TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
|
||||
TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
|
||||
LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
|
||||
|
||||
# credential overrides
|
||||
DANGEROUS_SALT = "development-notify-salt"
|
||||
SECRET_KEY = (
|
||||
"dev-notify-secret-key" # nosec B105 - this is only used in development
|
||||
)
|
||||
INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ["dev-notify-secret-key"]}
|
||||
ALLOW_EXPIRED_API_TOKEN = getenv("ALLOW_EXPIRED_API_TOKEN", "0") == "1"
|
||||
ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
|
||||
SECRET_KEY = 'dev-notify-secret-key'
|
||||
DANGEROUS_SALT = 'dev-notify-salt'
|
||||
|
||||
MMG_INBOUND_SMS_AUTH = ['testkey']
|
||||
MMG_INBOUND_SMS_USERNAME = ['username']
|
||||
|
||||
NOTIFY_ENVIRONMENT = 'development'
|
||||
NOTIFY_LOG_PATH = 'application.log'
|
||||
NOTIFICATION_QUEUE_PREFIX = 'development'
|
||||
NOTIFY_EMAIL_DOMAIN = "notify.tools"
|
||||
|
||||
SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
|
||||
REDIS_URL = 'redis://localhost:6379/0'
|
||||
|
||||
ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
|
||||
|
||||
for queue in QueueNames.all_queues():
|
||||
Config.CELERY_QUEUES.append(
|
||||
Queue(queue, Exchange('default'), routing_key=queue)
|
||||
)
|
||||
|
||||
API_HOST_NAME = "http://localhost:6011"
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
|
||||
|
||||
class Test(Development):
|
||||
FROM_NUMBER = "testing"
|
||||
NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
|
||||
FROM_NUMBER = 'testing'
|
||||
NOTIFY_ENVIRONMENT = 'test'
|
||||
TESTING = True
|
||||
|
||||
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
|
||||
TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
|
||||
TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
|
||||
LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
|
||||
|
||||
# this is overriden in jenkins and on cloudfoundry
|
||||
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
|
||||
|
||||
BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
|
||||
|
||||
ANTIVIRUS_ENABLED = True
|
||||
DVLA_EMAIL_ADDRESSES = [
|
||||
"success@simulator.amazonses.com",
|
||||
"success+2@simulator.amazonses.com",
|
||||
]
|
||||
|
||||
HIGH_VOLUME_SERVICE = [
|
||||
"941b6f9a-50d7-4742-8d50-f365ca74bf27",
|
||||
"63f95b86-2d19-4497-b8b2-ccf25457df4e",
|
||||
"7e5950cb-9954-41f5-8376-962b8c8555cf",
|
||||
"10d1b9c9-0072-4fa9-ae1c-595e333841da",
|
||||
]
|
||||
for queue in QueueNames.all_queues():
|
||||
Config.CELERY_QUEUES.append(
|
||||
Queue(queue, Exchange('default'), routing_key=queue)
|
||||
)
|
||||
|
||||
# this is overriden in CI
|
||||
SQLALCHEMY_DATABASE_URI = getenv("SQLALCHEMY_DATABASE_TEST_URI")
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
API_HOST_NAME = "http://localhost:6011"
|
||||
|
||||
CELERY = {
|
||||
**Config.CELERY,
|
||||
"broker_url": "you-forgot-to-mock-celery-in-your-tests://",
|
||||
}
|
||||
SMS_INBOUND_WHITELIST = ['203.0.113.195']
|
||||
FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
|
||||
TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
|
||||
|
||||
MMG_URL = 'https://example.com/mmg'
|
||||
FIRETEXT_URL = 'https://example.com/firetext'
|
||||
|
||||
|
||||
class Production(Config):
|
||||
# buckets
|
||||
CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
|
||||
f"notify-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}"
|
||||
)
|
||||
|
||||
FROM_NUMBER = "Notify.gov"
|
||||
class Preview(Config):
|
||||
NOTIFY_EMAIL_DOMAIN = 'notify.works'
|
||||
NOTIFY_ENVIRONMENT = 'preview'
|
||||
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
|
||||
TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
|
||||
TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
|
||||
LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
|
||||
FROM_NUMBER = 'preview'
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
CHECK_PROXY_HEADER = False
|
||||
|
||||
|
||||
class Staging(Production):
|
||||
class Staging(Config):
|
||||
NOTIFY_EMAIL_DOMAIN = 'staging-notify.works'
|
||||
NOTIFY_ENVIRONMENT = 'staging'
|
||||
CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
|
||||
TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
|
||||
TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
|
||||
LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
|
||||
FROM_NUMBER = 'stage'
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
CHECK_PROXY_HEADER = True
|
||||
REDIS_ENABLED = True
|
||||
|
||||
|
||||
class Live(Config):
|
||||
NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
|
||||
NOTIFY_ENVIRONMENT = 'live'
|
||||
CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
|
||||
TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
|
||||
TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
|
||||
LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
|
||||
FROM_NUMBER = 'GOVUK'
|
||||
PERFORMANCE_PLATFORM_ENABLED = True
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
CHECK_PROXY_HEADER = True
|
||||
|
||||
CRONITOR_ENABLED = True
|
||||
|
||||
|
||||
class CloudFoundryConfig(Config):
|
||||
pass
|
||||
|
||||
|
||||
class Demo(Production):
|
||||
pass
|
||||
# CloudFoundry sandbox
|
||||
class Sandbox(CloudFoundryConfig):
|
||||
NOTIFY_EMAIL_DOMAIN = 'notify.works'
|
||||
NOTIFY_ENVIRONMENT = 'sandbox'
|
||||
CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
|
||||
LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
|
||||
TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
|
||||
LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
|
||||
LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
|
||||
INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
|
||||
FROM_NUMBER = 'sandbox'
|
||||
REDIS_ENABLED = False
|
||||
|
||||
|
||||
configs = {
|
||||
"development": Development,
|
||||
"test": Test,
|
||||
"staging": Staging,
|
||||
"demo": Demo,
|
||||
"sandbox": Staging,
|
||||
"production": Production,
|
||||
'development': Development,
|
||||
'test': Test,
|
||||
'live': Live,
|
||||
'production': Live,
|
||||
'staging': Staging,
|
||||
'preview': Preview,
|
||||
'sandbox': Sandbox
|
||||
}
|
||||
|
||||
@@ -1,316 +0,0 @@
|
||||
[
|
||||
{
|
||||
"id": "42a23d19-504e-49bb-a95e-4976baff4757",
|
||||
"name": "Example text message template",
|
||||
"type": "sms",
|
||||
"subject": "",
|
||||
"content": [
|
||||
"Hi, I’m trying out Notify.gov. Today is ((day of week)) and my favorite color is ((color))."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
|
||||
"name": "Notify invitation email",
|
||||
"type": "email",
|
||||
"subject": "((user_name)) has invited you to collaborate on ((service_name)) on Notify.gov",
|
||||
"content": [
|
||||
"((user_name)) has invited you to collaborate on ((service_name)) on Notify.gov.",
|
||||
"",
|
||||
"",
|
||||
"Notify.gov makes it easy to keep people updated by helping you send text messages.",
|
||||
"",
|
||||
"",
|
||||
"If you have not done so, please log out before joining.",
|
||||
"",
|
||||
"[Join Service](((url)))",
|
||||
"If you’re new to Notify.gov you will first be directed to Login.gov create an account with us.",
|
||||
"",
|
||||
"",
|
||||
"This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure.",
|
||||
"",
|
||||
"",
|
||||
"Notify.gov uses Login.gov to allow you to sign in safely. Login.gov is a secure sign in service to help you access participating government agencies' digital touchpoints."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
|
||||
"name": "Notify SMS verify code",
|
||||
"type": "sms",
|
||||
"subject": "",
|
||||
"content": ["((verify_code)) is your Notify.gov authentication code."]
|
||||
},
|
||||
{
|
||||
"id": "474e9242-823b-4f99-813d-ed392e7f1201",
|
||||
"name": "Notify password reset email",
|
||||
"type": "email",
|
||||
"subject": "Reset your Notify.gov password",
|
||||
"content": [
|
||||
"Hi ((user_name)),",
|
||||
"",
|
||||
"",
|
||||
"We received a request to reset your password on Notify.gov.",
|
||||
"",
|
||||
"",
|
||||
"If you didn’t request this email, you can ignore it – your password has not been changed.",
|
||||
"",
|
||||
"",
|
||||
"To reset your password, click this link:",
|
||||
"",
|
||||
"",
|
||||
"((url))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "299726d2-dba6-42b8-8209-30e1d66ea164",
|
||||
"name": "Notify email verify code",
|
||||
"type": "email",
|
||||
"subject": "Sign in to Notify.gov",
|
||||
"content": [
|
||||
"Hi ((name)),",
|
||||
"",
|
||||
"",
|
||||
"To sign in to Notify.gov please open this link:",
|
||||
"",
|
||||
"",
|
||||
"((url))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
|
||||
"name": "Notify email verification code",
|
||||
"type": "email",
|
||||
"subject": "Confirm Notify.gov registration",
|
||||
"content": [
|
||||
"Hi ((name)),",
|
||||
"",
|
||||
"",
|
||||
"To complete your registration for Notify.gov please click the link below",
|
||||
"",
|
||||
"",
|
||||
"[Join Service](((url)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
|
||||
"name": "Your Notify.gov account",
|
||||
"type": "email",
|
||||
"subject": "Your Notify.gov account",
|
||||
"content": [
|
||||
"You already have a Notify.gov account with this email address.",
|
||||
"",
|
||||
"",
|
||||
"Sign in here: ((signin_url))",
|
||||
"",
|
||||
"",
|
||||
"If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"If you didn’t try to register for a Notify.gov account recently, please let us know here: ((feedback_url))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "eb4d9930-87ab-4aef-9bce-786762687884",
|
||||
"name": "Confirm new email address",
|
||||
"type": "email",
|
||||
"subject": "Confirm your email address for Notify.gov",
|
||||
"content": [
|
||||
"Hi ((name)),",
|
||||
"",
|
||||
"",
|
||||
"Click this link to confirm your new email address:",
|
||||
"",
|
||||
"",
|
||||
"((url))",
|
||||
"",
|
||||
"",
|
||||
"If you didn’t try to change the email address for your Notify.gov account, let us know here:",
|
||||
"",
|
||||
"",
|
||||
"((feedback_url))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
|
||||
"name": "Automated \"You’re now live\" message",
|
||||
"type": "email",
|
||||
"subject": "((service name)) is now live on Notify.gov",
|
||||
"content": [
|
||||
"Hi ((name)),",
|
||||
"",
|
||||
"",
|
||||
"Congratulations, we have received and reviewed your Go Live submission form. ((service name)) is now live on Notify.gov. With your Notify account you now can send messages for the use case outlined in your Go Live submission form.",
|
||||
"",
|
||||
"",
|
||||
"If you have any questions or experience any technical issues, please contact your pilot lead.",
|
||||
"",
|
||||
"",
|
||||
"Thanks",
|
||||
"",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "203566f0-d835-47c5-aa06-932439c86573",
|
||||
"name": "Notify organization invitation email",
|
||||
"type": "email",
|
||||
"subject": "((user_name)) has invited you to collaborate on ((organization_name)) on Notify.gov",
|
||||
"content": [
|
||||
"((user_name)) has invited you to collaborate on ((organization_name)) on Notify.gov.",
|
||||
"",
|
||||
"",
|
||||
"Notify.gov makes it easy to keep people updated by helping you send text messages and emails.",
|
||||
"",
|
||||
"",
|
||||
"Open this link to create an account on Notify.gov:",
|
||||
"",
|
||||
"[Join Organization](((url)))",
|
||||
"",
|
||||
"",
|
||||
"This invitation will stop working at midnight tomorrow. This is to keep ((organization_name)) secure."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
|
||||
"name": "Email address changed by service manager",
|
||||
"type": "email",
|
||||
"subject": "Your Notify.gov email address has changed",
|
||||
"content": [
|
||||
"Dear ((name)),",
|
||||
"",
|
||||
"",
|
||||
"((servicemanagername)) changed your Notify account email address to:",
|
||||
"",
|
||||
"",
|
||||
"((email address))",
|
||||
"",
|
||||
"",
|
||||
"You’ll need to use this email address next time you sign in.",
|
||||
"",
|
||||
"",
|
||||
"Thanks",
|
||||
"",
|
||||
"",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "8a31520f-4751-4789-8ea1-fe54496725eb",
|
||||
"name": "Phone number changed by service manager",
|
||||
"type": "sms",
|
||||
"subject": "",
|
||||
"content": [
|
||||
"Your mobile number was changed by ((servicemanagername)). Next time you sign in, your Notify.gov authentication code will be sent to this phone."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
|
||||
"name": "Verify email reply-to address for a service",
|
||||
"type": "email",
|
||||
"subject": "Your Notify.gov reply-to email address",
|
||||
"content": [
|
||||
"Hi,",
|
||||
"",
|
||||
"",
|
||||
"This address has been provided as a reply-to email address for a Notify.gov account.",
|
||||
"",
|
||||
"Any replies from users to emails they receive through Notify.gov will come back to this email address.",
|
||||
"",
|
||||
"",
|
||||
"This is just a quick check to make sure the address is valid.",
|
||||
"",
|
||||
"",
|
||||
"No need to reply.",
|
||||
"",
|
||||
"",
|
||||
"Thanks",
|
||||
"",
|
||||
"",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
|
||||
"name": "MOU Signed By Receipt",
|
||||
"type": "email",
|
||||
"subject": "You’ve accepted the Notify.gov data sharing and financial agreement",
|
||||
"content": [
|
||||
"Hi ((signed_by_name)),",
|
||||
"",
|
||||
"((org_name)) has accepted the Notify.gov data sharing and financial agreement. ",
|
||||
"",
|
||||
"If you need another copy of the agreement you can download it here: ((mou_link))",
|
||||
"",
|
||||
"",
|
||||
"Thanks,",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
|
||||
"name": "MOU Signed On Behalf Of Receipt - Signed by",
|
||||
"type": "email",
|
||||
"subject": "You’ve accepted the Notify.gov data sharing and financial agreement",
|
||||
"content": [
|
||||
"Hi ((signed_by_name)),",
|
||||
"",
|
||||
"((org_name)) has accepted the Notify.gov data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
|
||||
"",
|
||||
"If you need another copy of the agreement you can download it here: ((mou_link))",
|
||||
"",
|
||||
"",
|
||||
"Thanks,",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "522b6657-5ca5-4368-a294-6b527703bd0b",
|
||||
"name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
|
||||
"type": "email",
|
||||
"subject": "((org_name)) has accepted the Notify.gov data sharing and financial agreement",
|
||||
"content": [
|
||||
"Hi ((on_behalf_of_name)),",
|
||||
"",
|
||||
"((signed_by_name)) has accepted the Notify.gov data sharing and financial agreement on your behalf, for ((org_name)).",
|
||||
"",
|
||||
"Notify.gov lets teams in the public sector send text messages and emails. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
|
||||
"",
|
||||
"If you need another copy of the agreement you can download it here: ((mou_link))",
|
||||
"",
|
||||
"",
|
||||
"Thanks,",
|
||||
"Notify.gov team"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
|
||||
"name": "MOU Signed Notify Team Alert",
|
||||
"type": "email",
|
||||
"subject": "Someone signed an MOU for an org on Notify",
|
||||
"content": [
|
||||
"What’s up Notifiers,",
|
||||
"",
|
||||
"((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
|
||||
"",
|
||||
"See how ((org_name)) is using Notify here: ((org_dashboard_link))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
|
||||
"name": "Notify daily letter volumes",
|
||||
"type": "email",
|
||||
"subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
|
||||
"content": [
|
||||
"((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
|
||||
"",
|
||||
"((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
|
||||
"((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
|
||||
"((international_volume)) international letters (((international_sheets)) sheets).",
|
||||
"",
|
||||
"Thanks",
|
||||
"",
|
||||
"Notify.gov team"
|
||||
]
|
||||
}
|
||||
]
|
||||
52
app/cronitor.py
Normal file
52
app/cronitor.py
Normal file
@@ -0,0 +1,52 @@
|
||||
import requests
|
||||
from functools import wraps
|
||||
from flask import current_app
|
||||
|
||||
|
||||
def cronitor(task_name):
|
||||
# check if task_name is in config
|
||||
def decorator(func):
|
||||
def ping_cronitor(command):
|
||||
if not current_app.config['CRONITOR_ENABLED']:
|
||||
return
|
||||
|
||||
task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
|
||||
if not task_slug:
|
||||
current_app.logger.error(
|
||||
'Cronitor enabled but task_name {} not found in environment'.format(task_name)
|
||||
)
|
||||
return
|
||||
|
||||
if command not in {'run', 'complete', 'fail'}:
|
||||
raise ValueError('command {} not a valid cronitor command'.format(command))
|
||||
|
||||
try:
|
||||
resp = requests.get(
|
||||
'https://cronitor.link/{}/{}'.format(task_slug, command),
|
||||
# cronitor limits msg to 1000 characters
|
||||
params={
|
||||
'host': current_app.config['API_HOST_NAME'],
|
||||
}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
current_app.logger.warning('Cronitor API failed for task {} due to {}'.format(
|
||||
task_name,
|
||||
repr(e)
|
||||
))
|
||||
|
||||
@wraps(func)
|
||||
def inner_decorator(*args, **kwargs):
|
||||
ping_cronitor('run')
|
||||
try:
|
||||
ret = func(*args, **kwargs)
|
||||
status = 'complete'
|
||||
return ret
|
||||
except Exception:
|
||||
status = 'fail'
|
||||
raise
|
||||
finally:
|
||||
ping_cronitor(status)
|
||||
|
||||
return inner_decorator
|
||||
return decorator
|
||||
@@ -1,5 +1,4 @@
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from app import db
|
||||
|
||||
|
||||
@@ -9,6 +8,7 @@ class DAOException(SQLAlchemyError):
|
||||
|
||||
|
||||
class DAOClass(object):
|
||||
|
||||
class Meta:
|
||||
model = None
|
||||
|
||||
@@ -17,6 +17,13 @@ class DAOClass(object):
|
||||
if _commit:
|
||||
db.session.commit()
|
||||
|
||||
def update_instance(self, inst, update_dict, _commit=True):
|
||||
# Make sure the id is not included in the update_dict
|
||||
update_dict.pop('id')
|
||||
self.Meta.model.query.filter_by(id=inst.id).update(update_dict)
|
||||
if _commit:
|
||||
db.session.commit()
|
||||
|
||||
def delete_instance(self, inst, _commit=True):
|
||||
db.session.delete(inst)
|
||||
if _commit:
|
||||
|
||||
@@ -1,118 +1,51 @@
|
||||
from flask import current_app
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.date_util import get_current_calendar_year_start_year
|
||||
from app.enums import OrganizationType
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import AnnualBilling
|
||||
from app.dao.date_util import get_current_financial_year_start_year
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_create_or_update_annual_billing_for_year(
|
||||
service_id, free_sms_fragment_limit, financial_year_start
|
||||
):
|
||||
@transactional
|
||||
def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
|
||||
result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
|
||||
|
||||
if result:
|
||||
result.free_sms_fragment_limit = free_sms_fragment_limit
|
||||
else:
|
||||
result = AnnualBilling(
|
||||
service_id=service_id,
|
||||
financial_year_start=financial_year_start,
|
||||
free_sms_fragment_limit=free_sms_fragment_limit,
|
||||
)
|
||||
result = AnnualBilling(service_id=service_id, financial_year_start=financial_year_start,
|
||||
free_sms_fragment_limit=free_sms_fragment_limit)
|
||||
db.session.add(result)
|
||||
return result
|
||||
|
||||
|
||||
def dao_get_annual_billing(service_id):
|
||||
stmt = (
|
||||
select(AnnualBilling)
|
||||
.where(
|
||||
AnnualBilling.service_id == service_id,
|
||||
)
|
||||
.order_by(AnnualBilling.financial_year_start)
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return AnnualBilling.query.filter_by(
|
||||
service_id=service_id,
|
||||
).order_by(AnnualBilling.financial_year_start).all()
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_update_annual_billing_for_future_years(
|
||||
service_id, free_sms_fragment_limit, financial_year_start
|
||||
):
|
||||
stmt = (
|
||||
update(AnnualBilling)
|
||||
.where(
|
||||
AnnualBilling.service_id == service_id,
|
||||
AnnualBilling.financial_year_start > financial_year_start,
|
||||
)
|
||||
.values({"free_sms_fragment_limit": free_sms_fragment_limit})
|
||||
@transactional
|
||||
def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
|
||||
AnnualBilling.query.filter(
|
||||
AnnualBilling.service_id == service_id,
|
||||
AnnualBilling.financial_year_start > financial_year_start
|
||||
).update(
|
||||
{'free_sms_fragment_limit': free_sms_fragment_limit}
|
||||
)
|
||||
db.session.execute(stmt)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start=None):
|
||||
if not financial_year_start:
|
||||
financial_year_start = get_current_calendar_year_start_year()
|
||||
|
||||
stmt = select(AnnualBilling).where(
|
||||
AnnualBilling.service_id == service_id,
|
||||
AnnualBilling.financial_year_start == financial_year_start,
|
||||
)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
if not financial_year_start:
|
||||
financial_year_start = get_current_financial_year_start_year()
|
||||
|
||||
return AnnualBilling.query.filter_by(
|
||||
service_id=service_id,
|
||||
financial_year_start=financial_year_start
|
||||
).first()
|
||||
|
||||
|
||||
def dao_get_all_free_sms_fragment_limit(service_id):
|
||||
stmt = (
|
||||
select(AnnualBilling)
|
||||
.where(
|
||||
AnnualBilling.service_id == service_id,
|
||||
)
|
||||
.order_by(AnnualBilling.financial_year_start)
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
|
||||
|
||||
def set_default_free_allowance_for_service(service, year_start=None):
|
||||
default_free_sms_fragment_limits = {
|
||||
OrganizationType.FEDERAL: {
|
||||
2020: 250_000,
|
||||
2021: 150_000,
|
||||
2022: 40_000,
|
||||
},
|
||||
OrganizationType.STATE: {
|
||||
2020: 250_000,
|
||||
2021: 150_000,
|
||||
2022: 40_000,
|
||||
},
|
||||
OrganizationType.OTHER: {
|
||||
2020: 250_000,
|
||||
2021: 150_000,
|
||||
2022: 40_000,
|
||||
},
|
||||
}
|
||||
if not year_start:
|
||||
year_start = get_current_calendar_year_start_year()
|
||||
# handle cases where the year is less than 2020 or greater than 2021
|
||||
if year_start < 2020:
|
||||
year_start = 2020
|
||||
if year_start > 2022:
|
||||
year_start = 2022
|
||||
if service.organization_type:
|
||||
free_allowance = default_free_sms_fragment_limits[service.organization_type][
|
||||
year_start
|
||||
]
|
||||
else:
|
||||
current_app.logger.info(
|
||||
f"no organization type for service {service.id}. Using other default of "
|
||||
f"{default_free_sms_fragment_limits['other'][year_start]}"
|
||||
)
|
||||
free_allowance = default_free_sms_fragment_limits[OrganizationType.OTHER][
|
||||
year_start
|
||||
]
|
||||
|
||||
return dao_create_or_update_annual_billing_for_year(
|
||||
service.id, free_allowance, year_start
|
||||
)
|
||||
return AnnualBilling.query.filter_by(
|
||||
service_id=service_id,
|
||||
).order_by(AnnualBilling.financial_year_start).all()
|
||||
|
||||
@@ -1,83 +1,49 @@
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit, version_class
|
||||
from app.models import ApiKey
|
||||
from app.utils import utc_now
|
||||
|
||||
from app.dao.dao_utils import (
|
||||
transactional,
|
||||
version_class
|
||||
)
|
||||
|
||||
from sqlalchemy import or_, func
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
@version_class(ApiKey)
|
||||
def save_model_api_key(api_key):
|
||||
if not api_key.id:
|
||||
api_key.id = (
|
||||
uuid.uuid4()
|
||||
) # must be set now so version history model can use same id
|
||||
api_key.id = uuid.uuid4() # must be set now so version history model can use same id
|
||||
api_key.secret = uuid.uuid4()
|
||||
db.session.add(api_key)
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
@version_class(ApiKey)
|
||||
def expire_api_key(service_id, api_key_id):
|
||||
api_key = (
|
||||
db.session.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.id == api_key_id, ApiKey.service_id == service_id
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
api_key.expiry_date = utc_now()
|
||||
api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
|
||||
api_key.expiry_date = datetime.utcnow()
|
||||
db.session.add(api_key)
|
||||
|
||||
|
||||
def get_model_api_keys(service_id, id=None):
|
||||
if id:
|
||||
return (
|
||||
db.session.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.id == id,
|
||||
ApiKey.service_id == service_id,
|
||||
ApiKey.expiry_date == None, # noqa
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
seven_days_ago = utc_now() - timedelta(days=7)
|
||||
return (
|
||||
db.session.execute(
|
||||
select(ApiKey).where(
|
||||
or_(
|
||||
ApiKey.expiry_date == None, # noqa
|
||||
func.date(ApiKey.expiry_date) > seven_days_ago, # noqa
|
||||
),
|
||||
ApiKey.service_id == service_id,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return ApiKey.query.filter_by(id=id, service_id=service_id, expiry_date=None).one()
|
||||
seven_days_ago = datetime.utcnow() - timedelta(days=7)
|
||||
return ApiKey.query.filter(
|
||||
or_(ApiKey.expiry_date == None, func.date(ApiKey.expiry_date) > seven_days_ago), # noqa
|
||||
ApiKey.service_id == service_id
|
||||
).all()
|
||||
|
||||
|
||||
def get_unsigned_secrets(service_id):
|
||||
"""
|
||||
This method can only be exposed to the Authentication of the api calls.
|
||||
"""
|
||||
api_keys = (
|
||||
db.session.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.service_id == service_id, ApiKey.expiry_date == None # noqa
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
api_keys = ApiKey.query.filter_by(service_id=service_id, expiry_date=None).all()
|
||||
keys = [x.secret for x in api_keys]
|
||||
return keys
|
||||
|
||||
@@ -86,13 +52,5 @@ def get_unsigned_secret(key_id):
|
||||
"""
|
||||
This method can only be exposed to the Authentication of the api calls.
|
||||
"""
|
||||
api_key = (
|
||||
db.session.execute(
|
||||
select(ApiKey).where(
|
||||
ApiKey.id == key_id, ApiKey.expiry_date == None # noqa
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
api_key = ApiKey.query.filter_by(id=key_id, expiry_date=None).one()
|
||||
return api_key.secret
|
||||
|
||||
@@ -1,51 +1,34 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from flask import current_app
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy import desc
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.inbound_sms_dao import Pagination
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import Complaint
|
||||
from app.utils import get_midnight_in_utc
|
||||
from app.utils import get_london_midnight_in_utc
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def save_complaint(complaint):
|
||||
db.session.add(complaint)
|
||||
|
||||
|
||||
def fetch_paginated_complaints(page=1):
|
||||
page_size = current_app.config["PAGE_SIZE"]
|
||||
total_count = db.session.scalar(select(func.count()).select_from(Complaint))
|
||||
offset = (page - 1) * page_size
|
||||
stmt = (
|
||||
select(Complaint)
|
||||
.order_by(desc(Complaint.created_at))
|
||||
.offset(offset)
|
||||
.limit(page_size)
|
||||
return Complaint.query.order_by(
|
||||
desc(Complaint.created_at)
|
||||
).paginate(
|
||||
page=page,
|
||||
per_page=current_app.config['PAGE_SIZE']
|
||||
)
|
||||
result = db.session.execute(stmt).scalars().all()
|
||||
pagination = Pagination(result, page=page, per_page=page_size, total=total_count)
|
||||
return pagination
|
||||
|
||||
|
||||
def fetch_complaints_by_service(service_id):
|
||||
stmt = (
|
||||
select(Complaint)
|
||||
.where(Complaint.service_id == service_id)
|
||||
.order_by(desc(Complaint.created_at))
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return Complaint.query.filter_by(service_id=service_id).order_by(desc(Complaint.created_at)).all()
|
||||
|
||||
|
||||
def fetch_count_of_complaints(start_date, end_date):
|
||||
start_date = get_midnight_in_utc(start_date)
|
||||
end_date = get_midnight_in_utc(end_date + timedelta(days=1))
|
||||
start_date = get_london_midnight_in_utc(start_date)
|
||||
end_date = get_london_midnight_in_utc(end_date + timedelta(days=1))
|
||||
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Complaint)
|
||||
.where(Complaint.created_at >= start_date, Complaint.created_at < end_date)
|
||||
)
|
||||
return db.session.execute(stmt).scalar() or 0
|
||||
return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count()
|
||||
|
||||
38
app/dao/daily_sorted_letter_dao.py
Normal file
38
app/dao/daily_sorted_letter_dao.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import DailySortedLetter
|
||||
|
||||
|
||||
def dao_get_daily_sorted_letter_by_billing_day(billing_day):
|
||||
return DailySortedLetter.query.filter_by(
|
||||
billing_day=billing_day
|
||||
).first()
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
|
||||
'''
|
||||
This uses the Postgres upsert to avoid race conditions when two threads try and insert
|
||||
at the same row. The excluded object refers to values that we tried to insert but were
|
||||
rejected.
|
||||
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
|
||||
'''
|
||||
table = DailySortedLetter.__table__
|
||||
stmt = insert(table).values(
|
||||
billing_day=new_daily_sorted_letter.billing_day,
|
||||
file_name=new_daily_sorted_letter.file_name,
|
||||
unsorted_count=new_daily_sorted_letter.unsorted_count,
|
||||
sorted_count=new_daily_sorted_letter.sorted_count)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[table.c.billing_day, table.c.file_name],
|
||||
set_={
|
||||
'unsorted_count': stmt.excluded.unsorted_count,
|
||||
'sorted_count': stmt.excluded.sorted_count,
|
||||
'updated_at': datetime.utcnow()
|
||||
}
|
||||
)
|
||||
db.session.connection().execute(stmt)
|
||||
@@ -1,43 +1,27 @@
|
||||
import itertools
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
|
||||
from app import db
|
||||
from app.history_meta import create_history
|
||||
|
||||
|
||||
def autocommit(func):
|
||||
def transactional(func):
|
||||
@wraps(func)
|
||||
def commit_or_rollback(*args, **kwargs):
|
||||
from flask import current_app
|
||||
try:
|
||||
res = func(*args, **kwargs)
|
||||
|
||||
if not db.session().in_nested_transaction():
|
||||
db.session.commit()
|
||||
|
||||
db.session.commit()
|
||||
return res
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
current_app.logger.error(e)
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
return commit_or_rollback
|
||||
|
||||
|
||||
@contextmanager
|
||||
def transaction():
|
||||
try:
|
||||
db.session.begin_nested()
|
||||
yield
|
||||
db.session.commit()
|
||||
class VersionOptions():
|
||||
|
||||
if not db.session().in_nested_transaction():
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
class VersionOptions:
|
||||
def __init__(self, model_class, history_class=None, must_write_history=True):
|
||||
self.model_class = model_class
|
||||
self.history_class = history_class
|
||||
@@ -45,34 +29,38 @@ class VersionOptions:
|
||||
|
||||
|
||||
def version_class(*version_options):
|
||||
|
||||
if len(version_options) == 1 and not isinstance(version_options[0], VersionOptions):
|
||||
version_options = (VersionOptions(version_options[0]),)
|
||||
|
||||
def versioned(func):
|
||||
@wraps(func)
|
||||
def record_version(*args, **kwargs):
|
||||
|
||||
func(*args, **kwargs)
|
||||
|
||||
session_objects = []
|
||||
|
||||
for version_option in version_options:
|
||||
tmp_session_objects = [
|
||||
(session_object, version_option.history_class)
|
||||
(
|
||||
session_object, version_option.history_class
|
||||
)
|
||||
for session_object in itertools.chain(
|
||||
db.session.new, db.session.dirty
|
||||
)
|
||||
if isinstance(session_object, version_option.model_class)
|
||||
if isinstance(
|
||||
session_object, version_option.model_class
|
||||
)
|
||||
]
|
||||
|
||||
if tmp_session_objects == [] and version_option.must_write_history:
|
||||
raise RuntimeError(
|
||||
(
|
||||
"Can't record history for {} "
|
||||
"(something in your code has casued the database to "
|
||||
"flush the session early so there's nothing to "
|
||||
"copy into the history table)"
|
||||
).format(version_option.model_class.__name__)
|
||||
)
|
||||
raise RuntimeError((
|
||||
'Can\'t record history for {} '
|
||||
'(something in your code has casued the database to '
|
||||
'flush the session early so there\'s nothing to '
|
||||
'copy into the history table)'
|
||||
).format(version_option.model_class.__name__))
|
||||
|
||||
session_objects += tmp_session_objects
|
||||
|
||||
@@ -82,15 +70,8 @@ def version_class(*version_options):
|
||||
)
|
||||
|
||||
return record_version
|
||||
|
||||
return versioned
|
||||
|
||||
|
||||
def dao_rollback():
|
||||
db.session.rollback()
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_save_object(obj):
|
||||
# add/update object in db
|
||||
db.session.add(obj)
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import calendar
|
||||
from datetime import date, datetime, time, timedelta
|
||||
from datetime import datetime, timedelta, date, time
|
||||
|
||||
from app.utils import utc_now
|
||||
from notifications_utils.timezones import convert_bst_to_utc
|
||||
import pytz
|
||||
|
||||
|
||||
def get_months_for_financial_year(year):
|
||||
return [
|
||||
month for month in (get_months_for_year(1, 13, year)) if month < datetime.now()
|
||||
convert_bst_to_utc(month) for month in (
|
||||
get_months_for_year(4, 13, year)
|
||||
+ get_months_for_year(1, 4, year + 1)
|
||||
)
|
||||
if convert_bst_to_utc(month) < datetime.now()
|
||||
]
|
||||
|
||||
|
||||
@@ -14,104 +18,57 @@ def get_months_for_year(start, end, year):
|
||||
return [datetime(year, month, 1) for month in range(start, end)]
|
||||
|
||||
|
||||
def get_calendar_year(year):
|
||||
return get_new_years(year), get_new_years(year + 1) - timedelta(microseconds=1)
|
||||
def get_financial_year(year):
|
||||
return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
|
||||
|
||||
|
||||
def get_calendar_year_dates(year):
|
||||
year_start_datetime, year_end_datetime = get_calendar_year(year)
|
||||
|
||||
return (year_start_datetime.date(), year_end_datetime.date())
|
||||
def get_current_financial_year():
|
||||
now = datetime.utcnow()
|
||||
current_month = int(now.strftime('%-m'))
|
||||
current_year = int(now.strftime('%Y'))
|
||||
year = current_year if current_month > 3 else current_year - 1
|
||||
return get_financial_year(year)
|
||||
|
||||
|
||||
def get_current_calendar_year():
|
||||
now = utc_now()
|
||||
current_year = int(now.strftime("%Y"))
|
||||
year = current_year
|
||||
return get_calendar_year(year)
|
||||
|
||||
|
||||
def get_new_years(year):
|
||||
return datetime(year, 1, 1, 0, 0, 0)
|
||||
def get_april_fools(year):
|
||||
"""
|
||||
This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC,
|
||||
the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone.
|
||||
:param year: the year to calculate the April 1, 00:00 BST for
|
||||
:return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
|
||||
"""
|
||||
return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
|
||||
tzinfo=None)
|
||||
|
||||
|
||||
def get_month_start_and_end_date_in_utc(month_year):
|
||||
"""
|
||||
This function return the start and date of the month_year as UTC,
|
||||
:param month_year: the datetime to calculate the start and end date for that month
|
||||
:return: start_date, end_date, month
|
||||
This function return the start and date of the month_year as UTC,
|
||||
:param month_year: the datetime to calculate the start and end date for that month
|
||||
:return: start_date, end_date, month
|
||||
"""
|
||||
import calendar
|
||||
|
||||
_, num_days = calendar.monthrange(month_year.year, month_year.month)
|
||||
first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
|
||||
last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
|
||||
return first_day, last_day
|
||||
return convert_bst_to_utc(first_day), convert_bst_to_utc(last_day)
|
||||
|
||||
|
||||
def get_current_calendar_year_start_year():
|
||||
def get_current_financial_year_start_year():
|
||||
now = datetime.now()
|
||||
financial_year_start = now.year
|
||||
start_date, end_date = get_calendar_year(now.year)
|
||||
start_date, end_date = get_financial_year(now.year)
|
||||
if now < start_date:
|
||||
financial_year_start = financial_year_start - 1
|
||||
return financial_year_start
|
||||
|
||||
|
||||
def get_calendar_year_for_datetime(start_date):
|
||||
if isinstance(start_date, date):
|
||||
def get_financial_year_for_datetime(start_date):
|
||||
if type(start_date) == date:
|
||||
start_date = datetime.combine(start_date, time.min)
|
||||
|
||||
year = int(start_date.strftime("%Y"))
|
||||
if start_date < get_new_years(year):
|
||||
year = int(start_date.strftime('%Y'))
|
||||
if start_date < get_april_fools(year):
|
||||
return year - 1
|
||||
else:
|
||||
return year
|
||||
|
||||
|
||||
def get_number_of_days_for_month(year, month):
|
||||
return calendar.monthrange(year, month)[1]
|
||||
|
||||
|
||||
def generate_date_range(start_date, end_date=None, days=0):
|
||||
if end_date:
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
try:
|
||||
yield current_date.date()
|
||||
except ValueError:
|
||||
pass
|
||||
current_date += timedelta(days=1)
|
||||
elif days > 0:
|
||||
end_date = start_date + timedelta(days=days)
|
||||
current_date = start_date
|
||||
while current_date < end_date:
|
||||
try:
|
||||
yield current_date.date()
|
||||
except ValueError:
|
||||
pass
|
||||
current_date += timedelta(days=1)
|
||||
else:
|
||||
return "An end_date or number of days must be specified"
|
||||
|
||||
|
||||
def generate_hourly_range(start_date, end_date=None, hours=0):
|
||||
if end_date:
|
||||
current_time = start_date
|
||||
while current_time <= end_date:
|
||||
try:
|
||||
yield current_time
|
||||
except ValueError:
|
||||
pass
|
||||
current_time += timedelta(hours=1)
|
||||
elif hours > 0:
|
||||
end_time = start_date + timedelta(hours=hours)
|
||||
current_time = start_date
|
||||
while current_time < end_time:
|
||||
try:
|
||||
yield current_time
|
||||
except ValueError:
|
||||
pass
|
||||
current_time += timedelta(hours=1)
|
||||
else:
|
||||
return "An end_date or number of hours must be specified"
|
||||
|
||||
@@ -1,40 +1,26 @@
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import EmailBranding
|
||||
|
||||
|
||||
def dao_get_email_branding_options():
|
||||
return db.session.execute(select(EmailBranding)).scalars().all()
|
||||
return EmailBranding.query.all()
|
||||
|
||||
|
||||
def dao_get_email_branding_by_id(email_branding_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(EmailBranding).where(EmailBranding.id == email_branding_id)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
return EmailBranding.query.filter_by(id=email_branding_id).one()
|
||||
|
||||
|
||||
def dao_get_email_branding_by_name(email_branding_name):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(EmailBranding).where(EmailBranding.name == email_branding_name)
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
return EmailBranding.query.filter_by(name=email_branding_name).first()
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_create_email_branding(email_branding):
|
||||
db.session.add(email_branding)
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_update_email_branding(email_branding, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(email_branding, key, value or None)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,63 +0,0 @@
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.sql.expression import case
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.models import FactProcessingTime
|
||||
from app.utils import utc_now
|
||||
|
||||
|
||||
@autocommit
|
||||
def insert_update_processing_time(processing_time):
|
||||
"""
|
||||
This uses the Postgres upsert to avoid race conditions when two threads try and insert
|
||||
at the same row. The excluded object refers to values that we tried to insert but were
|
||||
rejected.
|
||||
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
|
||||
"""
|
||||
table = FactProcessingTime.__table__
|
||||
stmt = insert(table).values(
|
||||
local_date=processing_time.local_date,
|
||||
messages_total=processing_time.messages_total,
|
||||
messages_within_10_secs=processing_time.messages_within_10_secs,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[table.c.local_date],
|
||||
set_={
|
||||
"messages_total": stmt.excluded.messages_total,
|
||||
"messages_within_10_secs": stmt.excluded.messages_within_10_secs,
|
||||
"updated_at": utc_now(),
|
||||
},
|
||||
)
|
||||
db.session.connection().execute(stmt)
|
||||
|
||||
|
||||
def get_processing_time_percentage_for_date_range(start_date, end_date):
|
||||
query = (
|
||||
select(
|
||||
FactProcessingTime.local_date.cast(db.Text).label("date"),
|
||||
FactProcessingTime.messages_total,
|
||||
FactProcessingTime.messages_within_10_secs,
|
||||
case(
|
||||
(
|
||||
FactProcessingTime.messages_total > 0,
|
||||
(
|
||||
(
|
||||
FactProcessingTime.messages_within_10_secs
|
||||
/ FactProcessingTime.messages_total.cast(db.Float)
|
||||
)
|
||||
* 100
|
||||
),
|
||||
),
|
||||
(FactProcessingTime.messages_total == 0, 100.0),
|
||||
).label("percentage"),
|
||||
)
|
||||
.where(
|
||||
FactProcessingTime.local_date >= start_date,
|
||||
FactProcessingTime.local_date <= end_date,
|
||||
)
|
||||
.order_by(FactProcessingTime.local_date)
|
||||
)
|
||||
|
||||
return db.session.execute(query).all()
|
||||
@@ -1,61 +1,47 @@
|
||||
from sqlalchemy import and_, select, update
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import InboundNumber
|
||||
|
||||
|
||||
def dao_get_inbound_numbers():
|
||||
stmt = select(InboundNumber).order_by(InboundNumber.updated_at)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return InboundNumber.query.order_by(InboundNumber.updated_at).all()
|
||||
|
||||
|
||||
def dao_get_available_inbound_numbers():
|
||||
stmt = select(InboundNumber).where(
|
||||
InboundNumber.active, InboundNumber.service_id.is_(None)
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return InboundNumber.query.filter(InboundNumber.active, InboundNumber.service_id.is_(None)).all()
|
||||
|
||||
|
||||
def dao_get_inbound_number_for_service(service_id):
|
||||
stmt = select(InboundNumber).where(InboundNumber.service_id == service_id)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
return InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
|
||||
|
||||
|
||||
def dao_get_inbound_number(inbound_number_id):
|
||||
stmt = select(InboundNumber).where(InboundNumber.id == inbound_number_id)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_set_inbound_number_to_service(service_id, inbound_number):
|
||||
inbound_number.service_id = service_id
|
||||
db.session.add(inbound_number)
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_set_inbound_number_active_flag(service_id, active):
|
||||
stmt = select(InboundNumber).where(InboundNumber.service_id == service_id)
|
||||
inbound_number = db.session.execute(stmt).scalars().first()
|
||||
inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
|
||||
inbound_number.active = active
|
||||
|
||||
db.session.add(inbound_number)
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_allocate_number_for_service(service_id, inbound_number_id):
|
||||
stmt = (
|
||||
update(InboundNumber)
|
||||
.where(
|
||||
and_(
|
||||
InboundNumber.id == inbound_number_id, # noqa
|
||||
InboundNumber.active == True, # noqa
|
||||
InboundNumber.service_id == None, # noqa
|
||||
)
|
||||
)
|
||||
.values({"service_id": service_id})
|
||||
updated = InboundNumber.query.filter_by(
|
||||
id=inbound_number_id,
|
||||
active=True,
|
||||
service_id=None
|
||||
).update(
|
||||
{"service_id": service_id}
|
||||
)
|
||||
result = db.session.execute(stmt)
|
||||
if result.rowcount == 0:
|
||||
if not updated:
|
||||
raise Exception("Inbound number: {} is not available".format(inbound_number_id))
|
||||
return db.session.get(InboundNumber, inbound_number_id)
|
||||
return InboundNumber.query.get(inbound_number_id)
|
||||
|
||||
@@ -1,113 +1,85 @@
|
||||
from flask import current_app
|
||||
from sqlalchemy import and_, delete, desc, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy import desc, and_
|
||||
from sqlalchemy.orm import aliased
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.enums import NotificationType
|
||||
from app.models import InboundSms, InboundSmsHistory, ServiceDataRetention
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
|
||||
from app.utils import midnight_n_days_ago
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_create_inbound_sms(inbound_sms):
|
||||
db.session.add(inbound_sms)
|
||||
|
||||
|
||||
def dao_get_inbound_sms_for_service(
|
||||
service_id, user_number=None, *, limit_days=None, limit=None
|
||||
):
|
||||
q = (
|
||||
select(InboundSms)
|
||||
.where(InboundSms.service_id == service_id)
|
||||
.order_by(InboundSms.created_at.desc())
|
||||
def dao_get_inbound_sms_for_service(service_id, user_number=None, *, limit_days=None, limit=None):
|
||||
q = InboundSms.query.filter(
|
||||
InboundSms.service_id == service_id
|
||||
).order_by(
|
||||
InboundSms.created_at.desc()
|
||||
)
|
||||
if limit_days is not None:
|
||||
start_date = midnight_n_days_ago(limit_days)
|
||||
q = q.where(InboundSms.created_at >= start_date)
|
||||
q = q.filter(InboundSms.created_at >= start_date)
|
||||
|
||||
if user_number:
|
||||
q = q.where(InboundSms.user_number == user_number)
|
||||
q = q.filter(InboundSms.user_number == user_number)
|
||||
|
||||
if limit:
|
||||
q = q.limit(limit)
|
||||
|
||||
return db.session.execute(q).scalars().all()
|
||||
return q.all()
|
||||
|
||||
|
||||
def dao_get_paginated_inbound_sms_for_service_for_public_api(
|
||||
service_id, older_than=None, page_size=None
|
||||
service_id,
|
||||
older_than=None,
|
||||
page_size=None
|
||||
):
|
||||
if page_size is None:
|
||||
page_size = current_app.config["PAGE_SIZE"]
|
||||
page_size = current_app.config['PAGE_SIZE']
|
||||
|
||||
filters = [InboundSms.service_id == service_id]
|
||||
|
||||
if older_than:
|
||||
older_than_created_at = (
|
||||
db.session.query(InboundSms.created_at)
|
||||
.where(InboundSms.id == older_than)
|
||||
.scalar_subquery()
|
||||
)
|
||||
older_than_created_at = db.session.query(
|
||||
InboundSms.created_at).filter(InboundSms.id == older_than).as_scalar()
|
||||
filters.append(InboundSms.created_at < older_than_created_at)
|
||||
|
||||
page = 1 # ?
|
||||
offset = (page - 1) * page_size
|
||||
# As part of the move to sqlalchemy 2.0, we do this manual pagination
|
||||
stmt = (
|
||||
select(InboundSms)
|
||||
.where(*filters)
|
||||
.order_by(desc(InboundSms.created_at))
|
||||
.limit(page_size)
|
||||
.offset(offset)
|
||||
)
|
||||
paginated_items = db.session.execute(stmt).scalars().all()
|
||||
total_items = db.session.execute(select(func.count()).where(*filters)).scalar() or 0
|
||||
pagination = Pagination(paginated_items, page, page_size, total_items)
|
||||
return pagination
|
||||
query = InboundSms.query.filter(*filters)
|
||||
|
||||
return query.order_by(desc(InboundSms.created_at)).paginate(
|
||||
per_page=page_size
|
||||
).items
|
||||
|
||||
|
||||
def dao_count_inbound_sms_for_service(service_id, limit_days):
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(InboundSms)
|
||||
.where(
|
||||
InboundSms.service_id == service_id,
|
||||
InboundSms.created_at >= midnight_n_days_ago(limit_days),
|
||||
)
|
||||
)
|
||||
result = db.session.execute(stmt).scalar()
|
||||
return result
|
||||
return InboundSms.query.filter(
|
||||
InboundSms.service_id == service_id,
|
||||
InboundSms.created_at >= midnight_n_days_ago(limit_days)
|
||||
).count()
|
||||
|
||||
|
||||
def _insert_inbound_sms_history(subquery, query_limit=10000):
|
||||
offset = 0
|
||||
subquery_select = select(subquery)
|
||||
inbound_sms_stmt = select(
|
||||
InboundSms.id,
|
||||
InboundSms.created_at,
|
||||
InboundSms.service_id,
|
||||
InboundSms.notify_number,
|
||||
InboundSms.provider_date,
|
||||
InboundSms.provider_reference,
|
||||
InboundSms.provider,
|
||||
).where(InboundSms.id.in_(subquery_select))
|
||||
|
||||
count_query = select(func.count()).select_from(inbound_sms_stmt.subquery())
|
||||
inbound_sms_count = db.session.execute(count_query).scalar() or 0
|
||||
inbound_sms_query = db.session.query(
|
||||
*[x.name for x in InboundSmsHistory.__table__.c]
|
||||
).filter(InboundSms.id.in_(subquery))
|
||||
inbound_sms_count = inbound_sms_query.count()
|
||||
|
||||
while offset < inbound_sms_count:
|
||||
statement = insert(InboundSmsHistory).from_select(
|
||||
InboundSmsHistory.__table__.c,
|
||||
inbound_sms_stmt.limit(query_limit).offset(offset),
|
||||
inbound_sms_query.limit(query_limit).offset(offset)
|
||||
)
|
||||
|
||||
statement = statement.on_conflict_do_nothing(
|
||||
constraint="inbound_sms_history_pkey"
|
||||
)
|
||||
db.session.execute(statement)
|
||||
db.session.commit()
|
||||
db.session.connection().execute(statement)
|
||||
|
||||
offset += query_limit
|
||||
|
||||
@@ -115,12 +87,14 @@ def _insert_inbound_sms_history(subquery, query_limit=10000):
|
||||
def _delete_inbound_sms(datetime_to_delete_from, query_filter):
|
||||
query_limit = 10000
|
||||
|
||||
subquery = (
|
||||
select(InboundSms.id)
|
||||
.where(InboundSms.created_at < datetime_to_delete_from, *query_filter)
|
||||
.limit(query_limit)
|
||||
.subquery()
|
||||
)
|
||||
subquery = db.session.query(
|
||||
InboundSms.id
|
||||
).filter(
|
||||
InboundSms.created_at < datetime_to_delete_from,
|
||||
*query_filter
|
||||
).limit(
|
||||
query_limit
|
||||
).subquery()
|
||||
|
||||
deleted = 0
|
||||
# set to nonzero just to enter the loop
|
||||
@@ -128,66 +102,56 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
|
||||
while number_deleted > 0:
|
||||
_insert_inbound_sms_history(subquery, query_limit=query_limit)
|
||||
|
||||
stmt = delete(InboundSms).where(InboundSms.id.in_(subquery))
|
||||
number_deleted = db.session.execute(stmt).rowcount
|
||||
db.session.commit()
|
||||
number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(synchronize_session='fetch')
|
||||
deleted += number_deleted
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
@autocommit
|
||||
@statsd(namespace="dao")
|
||||
@transactional
|
||||
def delete_inbound_sms_older_than_retention():
|
||||
current_app.logger.info(
|
||||
"Deleting inbound sms for services with flexible data retention"
|
||||
)
|
||||
current_app.logger.info('Deleting inbound sms for services with flexible data retention')
|
||||
|
||||
stmt = (
|
||||
select(ServiceDataRetention)
|
||||
.join(ServiceDataRetention.service)
|
||||
.where(ServiceDataRetention.notification_type == NotificationType.SMS)
|
||||
)
|
||||
flexible_data_retention = db.session.execute(stmt).scalars().all()
|
||||
flexible_data_retention = ServiceDataRetention.query.join(
|
||||
ServiceDataRetention.service,
|
||||
Service.inbound_number
|
||||
).filter(
|
||||
ServiceDataRetention.notification_type == SMS_TYPE
|
||||
).all()
|
||||
|
||||
deleted = 0
|
||||
|
||||
for f in flexible_data_retention:
|
||||
n_days_ago = midnight_n_days_ago(f.days_of_retention)
|
||||
|
||||
current_app.logger.info(
|
||||
"Deleting inbound sms for service id: {}".format(f.service_id)
|
||||
)
|
||||
deleted += _delete_inbound_sms(
|
||||
n_days_ago, query_filter=[InboundSms.service_id == f.service_id]
|
||||
)
|
||||
current_app.logger.info("Deleting inbound sms for service id: {}".format(f.service_id))
|
||||
deleted += _delete_inbound_sms(n_days_ago, query_filter=[InboundSms.service_id == f.service_id])
|
||||
|
||||
current_app.logger.info(
|
||||
"Deleting inbound sms for services without flexible data retention"
|
||||
)
|
||||
current_app.logger.info('Deleting inbound sms for services without flexible data retention')
|
||||
|
||||
seven_days_ago = midnight_n_days_ago(7)
|
||||
|
||||
deleted += _delete_inbound_sms(
|
||||
seven_days_ago,
|
||||
query_filter=[
|
||||
InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
|
||||
],
|
||||
)
|
||||
deleted += _delete_inbound_sms(seven_days_ago, query_filter=[
|
||||
InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
|
||||
])
|
||||
|
||||
current_app.logger.info("Deleted {} inbound sms".format(deleted))
|
||||
current_app.logger.info('Deleted {} inbound sms'.format(deleted))
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
def dao_get_inbound_sms_by_id(service_id, inbound_id):
|
||||
stmt = select(InboundSms).where(
|
||||
InboundSms.id == inbound_id, InboundSms.service_id == service_id
|
||||
)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
return InboundSms.query.filter_by(
|
||||
id=inbound_id,
|
||||
service_id=service_id
|
||||
).one()
|
||||
|
||||
|
||||
def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
|
||||
service_id, page, limit_days
|
||||
service_id,
|
||||
page,
|
||||
limit_days
|
||||
):
|
||||
"""
|
||||
This query starts from inbound_sms and joins on to itself to find the most recent row for each user_number.
|
||||
@@ -208,44 +172,24 @@ def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
|
||||
LIMIT 50 OFFSET :page
|
||||
"""
|
||||
t2 = aliased(InboundSms)
|
||||
q = (
|
||||
select(InboundSms)
|
||||
.outerjoin(
|
||||
t2,
|
||||
and_(
|
||||
InboundSms.user_number == t2.user_number,
|
||||
InboundSms.service_id == t2.service_id,
|
||||
InboundSms.created_at < t2.created_at,
|
||||
),
|
||||
q = db.session.query(
|
||||
InboundSms
|
||||
).outerjoin(
|
||||
t2,
|
||||
and_(
|
||||
InboundSms.user_number == t2.user_number,
|
||||
InboundSms.service_id == t2.service_id,
|
||||
InboundSms.created_at < t2.created_at,
|
||||
)
|
||||
.where(
|
||||
t2.id.is_(None), # noqa
|
||||
InboundSms.service_id == service_id,
|
||||
InboundSms.created_at >= midnight_n_days_ago(limit_days),
|
||||
)
|
||||
.order_by(InboundSms.created_at.desc())
|
||||
).filter(
|
||||
t2.id == None, # noqa
|
||||
InboundSms.service_id == service_id,
|
||||
InboundSms.created_at >= midnight_n_days_ago(limit_days)
|
||||
).order_by(
|
||||
InboundSms.created_at.desc()
|
||||
)
|
||||
result = db.session.execute(q).scalars().all()
|
||||
page_size = current_app.config["PAGE_SIZE"]
|
||||
offset = (page - 1) * page_size
|
||||
paginated_results = result[offset : offset + page_size]
|
||||
pagination = Pagination(paginated_results, page, page_size, len(result))
|
||||
return pagination
|
||||
|
||||
|
||||
# TODO remove this when billing dao PR is merged.
|
||||
class Pagination:
|
||||
def __init__(self, items, page, per_page, total):
|
||||
self.items = items
|
||||
self.page = page
|
||||
self.per_page = per_page
|
||||
self.total = total
|
||||
self.pages = (total + per_page - 1) // per_page
|
||||
self.prev_num = page - 1 if page > 1 else None
|
||||
self.next_num = page + 1 if page < self.pages else None
|
||||
|
||||
def has_next(self):
|
||||
return self.page < self.pages
|
||||
|
||||
def has_prev(self):
|
||||
return self.page > 1
|
||||
return q.paginate(
|
||||
page=page,
|
||||
per_page=current_app.config['PAGE_SIZE']
|
||||
)
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from app import db
|
||||
from app.models import InvitedOrganizationUser
|
||||
from app.utils import utc_now
|
||||
|
||||
from app.models import InvitedOrganisationUser
|
||||
|
||||
|
||||
def save_invited_org_user(invited_org_user):
|
||||
@@ -12,48 +9,21 @@ def save_invited_org_user(invited_org_user):
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def get_invited_org_user(organization_id, invited_org_user_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(InvitedOrganizationUser).where(
|
||||
InvitedOrganizationUser.organization_id == organization_id,
|
||||
InvitedOrganizationUser.id == invited_org_user_id,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
def get_invited_org_user(organisation_id, invited_org_user_id):
|
||||
return InvitedOrganisationUser.query.filter_by(organisation_id=organisation_id, id=invited_org_user_id).one()
|
||||
|
||||
|
||||
def get_invited_org_user_by_id(invited_org_user_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(InvitedOrganizationUser).where(
|
||||
InvitedOrganizationUser.id == invited_org_user_id
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
return InvitedOrganisationUser.query.filter_by(id=invited_org_user_id).one()
|
||||
|
||||
|
||||
def get_invited_org_users_for_organization(organization_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(InvitedOrganizationUser).where(
|
||||
InvitedOrganizationUser.organization_id == organization_id
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
def get_invited_org_users_for_organisation(organisation_id):
|
||||
return InvitedOrganisationUser.query.filter_by(organisation_id=organisation_id).all()
|
||||
|
||||
|
||||
def delete_org_invitations_created_more_than_two_days_ago():
|
||||
deleted = (
|
||||
db.session.query(InvitedOrganizationUser)
|
||||
.where(InvitedOrganizationUser.created_at <= utc_now() - timedelta(days=2))
|
||||
.delete()
|
||||
)
|
||||
deleted = db.session.query(InvitedOrganisationUser).filter(
|
||||
InvitedOrganisationUser.created_at <= datetime.utcnow() - timedelta(days=2)
|
||||
).delete()
|
||||
db.session.commit()
|
||||
return deleted
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from app import db
|
||||
from app.enums import InvitedUserStatus
|
||||
|
||||
from app.models import InvitedUser
|
||||
from app.utils import utc_now
|
||||
|
||||
|
||||
def save_invited_user(invited_user):
|
||||
@@ -13,48 +9,21 @@ def save_invited_user(invited_user):
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def get_invited_user_by_service_and_id(service_id, invited_user_id):
|
||||
|
||||
stmt = select(InvitedUser).where(
|
||||
InvitedUser.service_id == service_id,
|
||||
InvitedUser.id == invited_user_id,
|
||||
)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
|
||||
|
||||
def get_expired_invite_by_service_and_id(service_id, invited_user_id):
|
||||
stmt = select(InvitedUser).where(
|
||||
InvitedUser.service_id == service_id,
|
||||
InvitedUser.id == invited_user_id,
|
||||
InvitedUser.status == InvitedUserStatus.EXPIRED,
|
||||
)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
def get_invited_user(service_id, invited_user_id):
|
||||
return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()
|
||||
|
||||
|
||||
def get_invited_user_by_id(invited_user_id):
|
||||
stmt = select(InvitedUser).where(InvitedUser.id == invited_user_id)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
|
||||
|
||||
def get_expired_invited_users_for_service(service_id):
|
||||
# TODO why does this return all invited users?
|
||||
stmt = select(InvitedUser).where(InvitedUser.service_id == service_id)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return InvitedUser.query.filter_by(id=invited_user_id).one()
|
||||
|
||||
|
||||
def get_invited_users_for_service(service_id):
|
||||
stmt = select(InvitedUser).where(InvitedUser.service_id == service_id)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return InvitedUser.query.filter_by(service_id=service_id).all()
|
||||
|
||||
|
||||
def expire_invitations_created_more_than_two_days_ago():
|
||||
expired = (
|
||||
db.session.query(InvitedUser)
|
||||
.where(
|
||||
InvitedUser.created_at <= utc_now() - timedelta(days=2),
|
||||
InvitedUser.status.in_((InvitedUserStatus.PENDING,)),
|
||||
)
|
||||
.update({InvitedUser.status: InvitedUserStatus.EXPIRED})
|
||||
)
|
||||
def delete_invitations_created_more_than_two_days_ago():
|
||||
deleted = db.session.query(InvitedUser).filter(
|
||||
InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2)
|
||||
).delete()
|
||||
db.session.commit()
|
||||
return expired
|
||||
return deleted
|
||||
|
||||
@@ -1,107 +1,83 @@
|
||||
import os
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from flask import current_app
|
||||
from sqlalchemy import and_, asc, desc, func, select, update
|
||||
from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
|
||||
from notifications_utils.statsd_decorators import statsd
|
||||
from sqlalchemy import (
|
||||
asc,
|
||||
desc,
|
||||
func,
|
||||
and_
|
||||
)
|
||||
|
||||
from app import db
|
||||
from app.dao.pagination import Pagination
|
||||
from app.enums import JobStatus
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.dao.templates_dao import dao_get_template_by_id
|
||||
from app.utils import midnight_n_days_ago
|
||||
|
||||
from app.models import (
|
||||
FactNotificationStatus,
|
||||
Job,
|
||||
JOB_STATUS_FINISHED,
|
||||
JOB_STATUS_PENDING,
|
||||
JOB_STATUS_SCHEDULED,
|
||||
LETTER_TYPE,
|
||||
Notification,
|
||||
ServiceDataRetention,
|
||||
Template,
|
||||
ServiceDataRetention,
|
||||
NOTIFICATION_CREATED,
|
||||
NOTIFICATION_CANCELLED,
|
||||
JOB_STATUS_CANCELLED,
|
||||
FactNotificationStatus
|
||||
)
|
||||
from app.utils import midnight_n_days_ago, utc_now
|
||||
|
||||
|
||||
@statsd(namespace="dao")
|
||||
def dao_get_notification_outcomes_for_job(service_id, job_id):
|
||||
stmt = (
|
||||
select(func.count(Notification.status).label("count"), Notification.status)
|
||||
.where(Notification.service_id == service_id, Notification.job_id == job_id)
|
||||
.group_by(Notification.status)
|
||||
)
|
||||
notification_statuses = db.session.execute(stmt).all()
|
||||
notification_statuses = db.session.query(
|
||||
func.count(Notification.status).label('count'), Notification.status
|
||||
).filter(
|
||||
Notification.service_id == service_id,
|
||||
Notification.job_id == job_id
|
||||
).group_by(
|
||||
Notification.status
|
||||
).all()
|
||||
|
||||
if not notification_statuses:
|
||||
stmt = select(
|
||||
FactNotificationStatus.notification_count.label("count"),
|
||||
FactNotificationStatus.notification_status.label("status"),
|
||||
).where(
|
||||
notification_statuses = db.session.query(
|
||||
FactNotificationStatus.notification_count.label('count'),
|
||||
FactNotificationStatus.notification_status.label('status')
|
||||
).filter(
|
||||
FactNotificationStatus.service_id == service_id,
|
||||
FactNotificationStatus.job_id == job_id,
|
||||
)
|
||||
notification_statuses = db.session.execute(stmt).all()
|
||||
FactNotificationStatus.job_id == job_id
|
||||
).all()
|
||||
return notification_statuses
|
||||
|
||||
|
||||
def dao_get_job_by_service_id_and_job_id(service_id, job_id):
|
||||
stmt = select(Job).where(Job.service_id == service_id, Job.id == job_id)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
return Job.query.filter_by(service_id=service_id, id=job_id).one()
|
||||
|
||||
|
||||
def dao_get_unfinished_jobs():
|
||||
|
||||
stmt = select(Job).filter(Job.processing_finished.is_(None))
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
|
||||
|
||||
def dao_get_jobs_by_service_id(
|
||||
service_id,
|
||||
*,
|
||||
limit_days=None,
|
||||
page=1,
|
||||
page_size=50,
|
||||
statuses=None,
|
||||
):
|
||||
def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
|
||||
query_filter = [
|
||||
Job.service_id == service_id,
|
||||
Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],
|
||||
Job.original_file_name != current_app.config["ONE_OFF_MESSAGE_FILENAME"],
|
||||
Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
|
||||
Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
|
||||
]
|
||||
if limit_days is not None:
|
||||
query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
|
||||
if statuses is not None and statuses != [""]:
|
||||
query_filter.append(Job.job_status.in_(statuses))
|
||||
|
||||
total_items = db.session.execute(
|
||||
select(func.count()).select_from(Job).where(*query_filter)
|
||||
).scalar_one()
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
stmt = (
|
||||
select(Job)
|
||||
.where(*query_filter)
|
||||
.order_by(
|
||||
func.coalesce(Job.processing_started, Job.created_at).desc(), Job.id.desc()
|
||||
if statuses is not None and statuses != ['']:
|
||||
query_filter.append(
|
||||
Job.job_status.in_(statuses)
|
||||
)
|
||||
.limit(page_size)
|
||||
.offset(offset)
|
||||
)
|
||||
items = db.session.execute(stmt).scalars().all()
|
||||
return Pagination(items, page, page_size, total_items)
|
||||
|
||||
|
||||
def dao_get_scheduled_job_stats(
|
||||
service_id,
|
||||
):
|
||||
|
||||
stmt = select(
|
||||
func.count(Job.id),
|
||||
func.min(Job.scheduled_for),
|
||||
).where(
|
||||
Job.service_id == service_id,
|
||||
Job.job_status == JobStatus.SCHEDULED,
|
||||
)
|
||||
return db.session.execute(stmt).one()
|
||||
return Job.query \
|
||||
.filter(*query_filter) \
|
||||
.order_by(Job.processing_started.desc(), Job.created_at.desc()) \
|
||||
.paginate(page=page, per_page=page_size)
|
||||
|
||||
|
||||
def dao_get_job_by_id(job_id):
|
||||
stmt = select(Job).where(Job.id == job_id)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
return Job.query.filter_by(id=job_id).one()
|
||||
|
||||
|
||||
def dao_archive_job(job):
|
||||
@@ -118,19 +94,17 @@ def dao_set_scheduled_jobs_to_pending():
|
||||
the transaction so that if the task is run more than once concurrently, one task will block the other select
|
||||
from completing until it commits.
|
||||
"""
|
||||
stmt = (
|
||||
select(Job)
|
||||
.where(
|
||||
Job.job_status == JobStatus.SCHEDULED,
|
||||
Job.scheduled_for < utc_now(),
|
||||
)
|
||||
.order_by(asc(Job.scheduled_for))
|
||||
.with_for_update()
|
||||
)
|
||||
jobs = db.session.execute(stmt).scalars().all()
|
||||
jobs = Job.query \
|
||||
.filter(
|
||||
Job.job_status == JOB_STATUS_SCHEDULED,
|
||||
Job.scheduled_for < datetime.utcnow()
|
||||
) \
|
||||
.order_by(asc(Job.scheduled_for)) \
|
||||
.with_for_update() \
|
||||
.all()
|
||||
|
||||
for job in jobs:
|
||||
job.job_status = JobStatus.PENDING
|
||||
job.job_status = JOB_STATUS_PENDING
|
||||
|
||||
db.session.add_all(jobs)
|
||||
db.session.commit()
|
||||
@@ -139,13 +113,14 @@ def dao_set_scheduled_jobs_to_pending():
|
||||
|
||||
|
||||
def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
|
||||
stmt = select(Job).where(
|
||||
Job.service_id == service_id,
|
||||
Job.id == job_id,
|
||||
Job.job_status == JobStatus.SCHEDULED,
|
||||
Job.scheduled_for > utc_now(),
|
||||
)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
return Job.query \
|
||||
.filter(
|
||||
Job.service_id == service_id,
|
||||
Job.id == job_id,
|
||||
Job.job_status == JOB_STATUS_SCHEDULED,
|
||||
Job.scheduled_for > datetime.utcnow()
|
||||
) \
|
||||
.one()
|
||||
|
||||
|
||||
def dao_create_job(job):
|
||||
@@ -153,25 +128,6 @@ def dao_create_job(job):
|
||||
job.id = uuid.uuid4()
|
||||
db.session.add(job)
|
||||
db.session.commit()
|
||||
# We are seeing weird time anomalies where a job can be created on
|
||||
# 8/19 yet show a created_at time of 8/16. This seems to be the only
|
||||
# place the created_at value is set so do some double-checking and debugging
|
||||
orig_time = job.created_at
|
||||
now_time = utc_now()
|
||||
diff_time = now_time - orig_time
|
||||
current_app.logger.warning(
|
||||
f"#notify-debug-admin-1859 dao_create_job orig created at {orig_time} and now {now_time}"
|
||||
)
|
||||
if diff_time.total_seconds() > 300: # It should be only a few seconds diff at most
|
||||
current_app.logger.warning(
|
||||
"#notify-debug-admin-1859 Something is wrong with job.created_at!"
|
||||
)
|
||||
if os.getenv("NOTIFY_ENVIRONMENT") not in ["test"]:
|
||||
job.created_at = now_time
|
||||
dao_update_job(job)
|
||||
current_app.logger.warning(
|
||||
f"#notify-debug-admin-1859 Job created_at reset to {job.created_at}"
|
||||
)
|
||||
|
||||
|
||||
def dao_update_job(job):
|
||||
@@ -179,94 +135,105 @@ def dao_update_job(job):
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def dao_update_job_status_to_error(job):
|
||||
stmt = update(Job).where(Job.id == job.id).values(job_status=JobStatus.ERROR)
|
||||
db.session.execute(stmt)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def dao_get_jobs_older_than_data_retention(notification_types):
|
||||
stmt = select(ServiceDataRetention).where(
|
||||
flexible_data_retention = ServiceDataRetention.query.filter(
|
||||
ServiceDataRetention.notification_type.in_(notification_types)
|
||||
)
|
||||
flexible_data_retention = db.session.execute(stmt).scalars().all()
|
||||
).all()
|
||||
jobs = []
|
||||
today = utc_now().date()
|
||||
today = datetime.utcnow().date()
|
||||
for f in flexible_data_retention:
|
||||
end_date = today - timedelta(days=f.days_of_retention)
|
||||
stmt = (
|
||||
select(Job)
|
||||
.join(Template)
|
||||
.where(
|
||||
func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
|
||||
Job.archived == False, # noqa
|
||||
Template.template_type == f.notification_type,
|
||||
Job.service_id == f.service_id,
|
||||
)
|
||||
.order_by(desc(Job.created_at))
|
||||
)
|
||||
jobs.extend(db.session.execute(stmt).scalars().all())
|
||||
|
||||
# notify-api-1287, make default data retention 7 days, 23 hours
|
||||
end_date = today - timedelta(days=7, hours=23)
|
||||
jobs.extend(Job.query.join(Template).filter(
|
||||
func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
|
||||
Job.archived == False, # noqa
|
||||
Template.template_type == f.notification_type,
|
||||
Job.service_id == f.service_id
|
||||
).order_by(desc(Job.created_at)).all())
|
||||
|
||||
end_date = today - timedelta(days=7)
|
||||
for notification_type in notification_types:
|
||||
services_with_data_retention = [
|
||||
x.service_id
|
||||
for x in flexible_data_retention
|
||||
if x.notification_type == notification_type
|
||||
x.service_id for x in flexible_data_retention if x.notification_type == notification_type
|
||||
]
|
||||
stmt = (
|
||||
select(Job)
|
||||
.join(Template)
|
||||
.where(
|
||||
func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
|
||||
Job.archived == False, # noqa
|
||||
Template.template_type == notification_type,
|
||||
Job.service_id.notin_(services_with_data_retention),
|
||||
)
|
||||
.order_by(desc(Job.created_at))
|
||||
)
|
||||
jobs.extend(db.session.execute(stmt).scalars().all())
|
||||
jobs.extend(Job.query.join(Template).filter(
|
||||
func.coalesce(Job.scheduled_for, Job.created_at) < end_date,
|
||||
Job.archived == False, # noqa
|
||||
Template.template_type == notification_type,
|
||||
Job.service_id.notin_(services_with_data_retention)
|
||||
).order_by(desc(Job.created_at)).all())
|
||||
|
||||
return jobs
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_cancel_letter_job(job):
|
||||
number_of_notifications_cancelled = Notification.query.filter(
|
||||
Notification.job_id == job.id
|
||||
).update({'status': NOTIFICATION_CANCELLED,
|
||||
'updated_at': datetime.utcnow(),
|
||||
'billable_units': 0})
|
||||
job.job_status = JOB_STATUS_CANCELLED
|
||||
dao_update_job(job)
|
||||
return number_of_notifications_cancelled
|
||||
|
||||
|
||||
def can_letter_job_be_cancelled(job):
|
||||
template = dao_get_template_by_id(job.template_id)
|
||||
if template.template_type != LETTER_TYPE:
|
||||
return False, "Only letter jobs can be cancelled through this endpoint. This is not a letter job."
|
||||
|
||||
notifications = Notification.query.filter(
|
||||
Notification.job_id == job.id
|
||||
).all()
|
||||
count_notifications = len(notifications)
|
||||
if job.job_status != JOB_STATUS_FINISHED or count_notifications != job.notification_count:
|
||||
return False, "We are still processing these letters, please try again in a minute."
|
||||
count_cancellable_notifications = len([
|
||||
n for n in notifications if n.status in CANCELLABLE_JOB_LETTER_STATUSES
|
||||
])
|
||||
if count_cancellable_notifications != job.notification_count or not letter_can_be_cancelled(
|
||||
NOTIFICATION_CREATED, job.created_at
|
||||
):
|
||||
return False, "It’s too late to cancel sending, these letters have already been sent."
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def find_jobs_with_missing_rows():
|
||||
# Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
|
||||
# Using 20 minutes as a condition seems reasonable.
|
||||
ten_minutes_ago = utc_now() - timedelta(minutes=20)
|
||||
yesterday = utc_now() - timedelta(days=1)
|
||||
jobs_with_rows_missing = (
|
||||
select(Job)
|
||||
.where(
|
||||
Job.job_status == JobStatus.FINISHED,
|
||||
Job.processing_finished < ten_minutes_ago,
|
||||
Job.processing_finished > yesterday,
|
||||
Job.id == Notification.job_id,
|
||||
)
|
||||
.group_by(Job)
|
||||
.having(func.count(Notification.id) != Job.notification_count)
|
||||
# Jobs can be a maximum of 50,000 rows. It typically takes 5 minutes to create all those notifications.
|
||||
# Using 10 minutes as a condition seems reasonable.
|
||||
ten_minutes_ago = datetime.utcnow() - timedelta(minutes=10)
|
||||
yesterday = datetime.utcnow() - timedelta(days=1)
|
||||
jobs_with_rows_missing = db.session.query(
|
||||
func.count(Notification.id).label('actual_count'),
|
||||
Job
|
||||
).filter(
|
||||
Job.job_status == JOB_STATUS_FINISHED,
|
||||
Job.processing_finished < ten_minutes_ago,
|
||||
Job.processing_finished > yesterday,
|
||||
Job.id == Notification.job_id,
|
||||
|
||||
).group_by(
|
||||
Job
|
||||
).having(
|
||||
func.count(Notification.id) != Job.notification_count
|
||||
)
|
||||
|
||||
return db.session.execute(jobs_with_rows_missing).scalars().all()
|
||||
return jobs_with_rows_missing.all()
|
||||
|
||||
|
||||
def find_missing_row_for_job(job_id, job_size):
|
||||
expected_row_numbers = select(
|
||||
func.generate_series(0, job_size - 1).label("row")
|
||||
expected_row_numbers = db.session.query(
|
||||
func.generate_series(0, job_size - 1).label('row')
|
||||
).subquery()
|
||||
|
||||
query = (
|
||||
select(
|
||||
Notification.job_row_number, expected_row_numbers.c.row.label("missing_row")
|
||||
)
|
||||
.outerjoin(
|
||||
Notification,
|
||||
and_(
|
||||
expected_row_numbers.c.row == Notification.job_row_number,
|
||||
Notification.job_id == job_id,
|
||||
),
|
||||
)
|
||||
.where(Notification.job_row_number == None) # noqa
|
||||
query = db.session.query(
|
||||
Notification.job_row_number,
|
||||
expected_row_numbers.c.row.label('missing_row')
|
||||
).outerjoin(
|
||||
Notification, and_(expected_row_numbers.c.row == Notification.job_row_number, Notification.job_id == job_id)
|
||||
).filter(
|
||||
Notification.job_row_number == None # noqa
|
||||
)
|
||||
return db.session.execute(query).all()
|
||||
return query.all()
|
||||
|
||||
29
app/dao/letter_branding_dao.py
Normal file
29
app/dao/letter_branding_dao.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from app import db
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import LetterBranding
|
||||
|
||||
|
||||
def dao_get_letter_branding_by_id(letter_branding_id):
|
||||
return LetterBranding.query.filter(LetterBranding.id == letter_branding_id).one()
|
||||
|
||||
|
||||
def dao_get_letter_branding_by_name(letter_branding_name):
|
||||
return LetterBranding.query.filter_by(name=letter_branding_name).first()
|
||||
|
||||
|
||||
def dao_get_all_letter_branding():
|
||||
return LetterBranding.query.order_by(LetterBranding.name).all()
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_create_letter_branding(letter_branding):
|
||||
db.session.add(letter_branding)
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_update_letter_branding(letter_branding_id, **kwargs):
|
||||
letter_branding = LetterBranding.query.get(letter_branding_id)
|
||||
for key, value in kwargs.items():
|
||||
setattr(letter_branding, key, value or None)
|
||||
db.session.add(letter_branding)
|
||||
return letter_branding
|
||||
File diff suppressed because it is too large
Load Diff
135
app/dao/organisation_dao.py
Normal file
135
app/dao/organisation_dao.py
Normal file
@@ -0,0 +1,135 @@
|
||||
from sqlalchemy.sql.expression import func
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import VersionOptions, transactional, version_class
|
||||
from app.models import (
|
||||
Organisation,
|
||||
Domain,
|
||||
InvitedOrganisationUser,
|
||||
Service,
|
||||
User
|
||||
)
|
||||
|
||||
|
||||
def dao_get_organisations():
|
||||
return Organisation.query.order_by(
|
||||
Organisation.active.desc(), Organisation.name.asc()
|
||||
).all()
|
||||
|
||||
|
||||
def dao_count_organsations_with_live_services():
|
||||
return db.session.query(Organisation.id).join(Organisation.services).filter(
|
||||
Service.active.is_(True),
|
||||
Service.restricted.is_(False),
|
||||
Service.count_as_live.is_(True),
|
||||
).distinct().count()
|
||||
|
||||
|
||||
def dao_get_organisation_services(organisation_id):
|
||||
return Organisation.query.filter_by(
|
||||
id=organisation_id
|
||||
).one().services
|
||||
|
||||
|
||||
def dao_get_organisation_by_id(organisation_id):
|
||||
return Organisation.query.filter_by(id=organisation_id).one()
|
||||
|
||||
|
||||
def dao_get_organisation_by_email_address(email_address):
|
||||
|
||||
email_address = email_address.lower().replace('.gsi.gov.uk', '.gov.uk')
|
||||
|
||||
for domain in Domain.query.order_by(func.char_length(Domain.domain).desc()).all():
|
||||
|
||||
if (
|
||||
email_address.endswith("@{}".format(domain.domain)) or
|
||||
email_address.endswith(".{}".format(domain.domain))
|
||||
):
|
||||
return Organisation.query.filter_by(id=domain.organisation_id).one()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def dao_get_organisation_by_service_id(service_id):
|
||||
return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_create_organisation(organisation):
|
||||
db.session.add(organisation)
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_update_organisation(organisation_id, **kwargs):
|
||||
|
||||
domains = kwargs.pop('domains', None)
|
||||
|
||||
num_updated = Organisation.query.filter_by(id=organisation_id).update(
|
||||
kwargs
|
||||
)
|
||||
|
||||
if isinstance(domains, list):
|
||||
|
||||
Domain.query.filter_by(organisation_id=organisation_id).delete()
|
||||
|
||||
db.session.bulk_save_objects([
|
||||
Domain(domain=domain.lower(), organisation_id=organisation_id)
|
||||
for domain in domains
|
||||
])
|
||||
|
||||
organisation = Organisation.query.get(organisation_id)
|
||||
|
||||
if 'organisation_type' in kwargs:
|
||||
_update_organisation_services(organisation, 'organisation_type', only_where_none=False)
|
||||
|
||||
if 'email_branding_id' in kwargs:
|
||||
_update_organisation_services(organisation, 'email_branding')
|
||||
|
||||
if 'letter_branding_id' in kwargs:
|
||||
_update_organisation_services(organisation, 'letter_branding')
|
||||
|
||||
return num_updated
|
||||
|
||||
|
||||
@version_class(
|
||||
VersionOptions(Service, must_write_history=False),
|
||||
)
|
||||
def _update_organisation_services(organisation, attribute, only_where_none=True):
|
||||
for service in organisation.services:
|
||||
if getattr(service, attribute) is None or not only_where_none:
|
||||
setattr(service, attribute, getattr(organisation, attribute))
|
||||
db.session.add(service)
|
||||
|
||||
|
||||
@transactional
|
||||
@version_class(Service)
|
||||
def dao_add_service_to_organisation(service, organisation_id):
|
||||
organisation = Organisation.query.filter_by(
|
||||
id=organisation_id
|
||||
).one()
|
||||
|
||||
service.organisation_id = organisation_id
|
||||
service.organisation_type = organisation.organisation_type
|
||||
service.crown = organisation.crown
|
||||
|
||||
db.session.add(service)
|
||||
|
||||
|
||||
def dao_get_invited_organisation_user(user_id):
|
||||
return InvitedOrganisationUser.query.filter_by(id=user_id).one()
|
||||
|
||||
|
||||
def dao_get_users_for_organisation(organisation_id):
|
||||
return User.query.filter(
|
||||
User.organisations.any(id=organisation_id),
|
||||
User.state == 'active'
|
||||
).order_by(User.created_at).all()
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_add_user_to_organisation(organisation_id, user_id):
|
||||
organisation = dao_get_organisation_by_id(organisation_id)
|
||||
user = User.query.filter_by(id=user_id).one()
|
||||
user.organisations.append(organisation)
|
||||
db.session.add(organisation)
|
||||
return user
|
||||
@@ -1,146 +0,0 @@
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.sql.expression import func
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import VersionOptions, autocommit, version_class
|
||||
from app.models import Domain, Organization, Service, User
|
||||
|
||||
|
||||
def dao_get_organizations():
|
||||
stmt = select(Organization).order_by(
|
||||
Organization.active.desc(), Organization.name.asc()
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
|
||||
|
||||
def dao_count_organizations_with_live_services():
|
||||
stmt = (
|
||||
select(func.count(func.distinct(Organization.id)))
|
||||
.join(Organization.services)
|
||||
.where(
|
||||
Service.active.is_(True),
|
||||
Service.restricted.is_(False),
|
||||
Service.count_as_live.is_(True),
|
||||
)
|
||||
)
|
||||
return db.session.execute(stmt).scalar() or 0
|
||||
|
||||
|
||||
def dao_get_organization_services(organization_id):
|
||||
stmt = select(Organization).where(Organization.id == organization_id)
|
||||
return db.session.execute(stmt).scalars().one().services
|
||||
|
||||
|
||||
def dao_get_organization_live_services(organization_id):
|
||||
stmt = select(Service).where(
|
||||
Service.organization_id == organization_id, Service.restricted == False # noqa
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
|
||||
|
||||
def dao_get_organization_by_id(organization_id):
|
||||
stmt = select(Organization).where(Organization.id == organization_id)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
|
||||
|
||||
def dao_get_organization_by_email_address(email_address):
|
||||
email_address = email_address.lower().replace(".gsi.gov.uk", ".gov.uk")
|
||||
stmt = select(Domain).order_by(func.char_length(Domain.domain).desc())
|
||||
domains = db.session.execute(stmt).scalars().all()
|
||||
for domain in domains:
|
||||
if email_address.endswith(
|
||||
"@{}".format(domain.domain)
|
||||
) or email_address.endswith(".{}".format(domain.domain)):
|
||||
stmt = select(Organization).where(Organization.id == domain.organization_id)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def dao_get_organization_by_service_id(service_id):
|
||||
stmt = (
|
||||
select(Organization).join(Organization.services).where(Service.id == service_id)
|
||||
)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_create_organization(organization):
|
||||
db.session.add(organization)
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_update_organization(organization_id, **kwargs):
|
||||
domains = kwargs.pop("domains", None)
|
||||
stmt = (
|
||||
update(Organization).where(Organization.id == organization_id).values(**kwargs)
|
||||
)
|
||||
num_updated = db.session.execute(stmt).rowcount
|
||||
|
||||
if isinstance(domains, list):
|
||||
stmt = delete(Domain).where(Domain.organization_id == organization_id)
|
||||
db.session.execute(stmt)
|
||||
db.session.bulk_save_objects(
|
||||
[
|
||||
Domain(domain=domain.lower(), organization_id=organization_id)
|
||||
for domain in domains
|
||||
]
|
||||
)
|
||||
|
||||
organization = db.session.get(Organization, organization_id)
|
||||
if "organization_type" in kwargs:
|
||||
_update_organization_services(
|
||||
organization, "organization_type", only_where_none=False
|
||||
)
|
||||
|
||||
if "email_branding_id" in kwargs:
|
||||
_update_organization_services(organization, "email_branding")
|
||||
|
||||
return num_updated
|
||||
|
||||
|
||||
@version_class(
|
||||
VersionOptions(Service, must_write_history=False),
|
||||
)
|
||||
def _update_organization_services(organization, attribute, only_where_none=True):
|
||||
for service in organization.services:
|
||||
if getattr(service, attribute) is None or not only_where_none:
|
||||
setattr(service, attribute, getattr(organization, attribute))
|
||||
db.session.add(service)
|
||||
|
||||
|
||||
@autocommit
|
||||
@version_class(Service)
|
||||
def dao_add_service_to_organization(service, organization_id):
|
||||
stmt = select(Organization).where(Organization.id == organization_id)
|
||||
organization = db.session.execute(stmt).scalars().one()
|
||||
|
||||
service.organization_id = organization_id
|
||||
service.organization_type = organization.organization_type
|
||||
|
||||
db.session.add(service)
|
||||
|
||||
|
||||
def dao_get_users_for_organization(organization_id):
|
||||
return (
|
||||
db.session.query(User)
|
||||
.join(User.organizations)
|
||||
.where(Organization.id == organization_id, User.state == "active")
|
||||
.order_by(User.created_at)
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_add_user_to_organization(organization_id, user_id):
|
||||
organization = dao_get_organization_by_id(organization_id)
|
||||
stmt = select(User).where(User.id == user_id)
|
||||
user = db.session.execute(stmt).scalars().one()
|
||||
user.organizations.append(organization)
|
||||
db.session.add(organization)
|
||||
return user
|
||||
|
||||
|
||||
@autocommit
|
||||
def dao_remove_user_from_organization(organization, user):
|
||||
organization.users.remove(user)
|
||||
@@ -1,15 +0,0 @@
|
||||
class Pagination:
|
||||
def __init__(self, items, page, per_page, total):
|
||||
self.items = items
|
||||
self.page = page
|
||||
self.per_page = per_page
|
||||
self.total = total
|
||||
self.pages = (total + per_page - 1) // per_page
|
||||
self.prev_num = page - 1 if page > 1 else None
|
||||
self.next_num = page + 1 if page < self.pages else None
|
||||
|
||||
def has_next(self):
|
||||
return self.page < self.pages
|
||||
|
||||
def has_prev(self):
|
||||
return self.page > 1
|
||||
@@ -1,44 +1,52 @@
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app import db
|
||||
from app.dao import DAOClass
|
||||
from app.enums import PermissionType
|
||||
from app.models import Permission, Service
|
||||
from app.models import (
|
||||
Permission,
|
||||
MANAGE_USERS,
|
||||
MANAGE_TEMPLATES,
|
||||
MANAGE_SETTINGS,
|
||||
SEND_TEXTS,
|
||||
SEND_EMAILS,
|
||||
SEND_LETTERS,
|
||||
MANAGE_API_KEYS,
|
||||
VIEW_ACTIVITY)
|
||||
|
||||
|
||||
# Default permissions for a service
|
||||
default_service_permissions = [
|
||||
MANAGE_USERS,
|
||||
MANAGE_TEMPLATES,
|
||||
MANAGE_SETTINGS,
|
||||
SEND_TEXTS,
|
||||
SEND_EMAILS,
|
||||
SEND_LETTERS,
|
||||
MANAGE_API_KEYS,
|
||||
VIEW_ACTIVITY]
|
||||
|
||||
|
||||
class PermissionDAO(DAOClass):
|
||||
|
||||
class Meta:
|
||||
model = Permission
|
||||
|
||||
def add_default_service_permissions_for_user(self, user, service):
|
||||
for name in PermissionType.defaults():
|
||||
for name in default_service_permissions:
|
||||
permission = Permission(permission=name, user=user, service=service)
|
||||
self.create_instance(permission, _commit=False)
|
||||
|
||||
def remove_user_service_permissions(self, user, service):
|
||||
db.session.execute(
|
||||
delete(self.Meta.model).where(
|
||||
self.Meta.model.user == user, self.Meta.model.service == service
|
||||
)
|
||||
)
|
||||
db.session.commit()
|
||||
query = self.Meta.model.query.filter_by(user=user, service=service)
|
||||
query.delete()
|
||||
|
||||
def remove_user_service_permissions_for_all_services(self, user):
|
||||
db.session.execute(delete(self.Meta.model).where(self.Meta.model.user == user))
|
||||
db.session.commit()
|
||||
query = self.Meta.model.query.filter_by(user=user)
|
||||
query.delete()
|
||||
|
||||
def set_user_service_permission(
|
||||
self, user, service, permissions, _commit=False, replace=False
|
||||
):
|
||||
def set_user_service_permission(self, user, service, permissions, _commit=False, replace=False):
|
||||
try:
|
||||
if replace:
|
||||
db.session.execute(
|
||||
delete(self.Meta.model).where(
|
||||
self.Meta.model.user == user, self.Meta.model.service == service
|
||||
)
|
||||
)
|
||||
|
||||
db.session.commit()
|
||||
query = self.Meta.model.query.filter_by(user=user, service=service)
|
||||
query.delete()
|
||||
for p in permissions:
|
||||
p.user = user
|
||||
p.service = service
|
||||
@@ -52,29 +60,12 @@ class PermissionDAO(DAOClass):
|
||||
db.session.commit()
|
||||
|
||||
def get_permissions_by_user_id(self, user_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(Permission)
|
||||
.join(Service)
|
||||
.where(Permission.user_id == user_id)
|
||||
.where(Service.active.is_(True))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return self.Meta.model.query.filter_by(user_id=user_id)\
|
||||
.join(Permission.service).filter_by(active=True).all()
|
||||
|
||||
def get_permissions_by_user_id_and_service_id(self, user_id, service_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(Permission)
|
||||
.join(Service)
|
||||
.where(Permission.user_id == user_id)
|
||||
.where(Service.active.is_(True))
|
||||
.where(Service.id == service_id)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return self.Meta.model.query.filter_by(user_id=user_id)\
|
||||
.join(Permission.service).filter_by(active=True, id=service_id).all()
|
||||
|
||||
|
||||
permission_dao = PermissionDAO()
|
||||
|
||||
@@ -1,39 +1,50 @@
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from notifications_utils.timezones import convert_utc_to_bst
|
||||
from sqlalchemy import asc, desc, func
|
||||
from flask import current_app
|
||||
from sqlalchemy import desc, func, select
|
||||
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.enums import NotificationType
|
||||
from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, User
|
||||
from app.utils import utc_now
|
||||
|
||||
|
||||
def get_provider_details_by_id(provider_details_id):
|
||||
return db.session.get(ProviderDetails, provider_details_id)
|
||||
return ProviderDetails.query.get(provider_details_id)
|
||||
|
||||
|
||||
def get_provider_details_by_identifier(identifier):
|
||||
stmt = select(ProviderDetails).where(ProviderDetails.identifier == identifier)
|
||||
return db.session.execute(stmt).scalars().one()
|
||||
return ProviderDetails.query.filter_by(identifier=identifier).one()
|
||||
|
||||
|
||||
def get_alternative_sms_provider(identifier):
|
||||
if identifier == "sns":
|
||||
raise Exception("No alternative SMS providers currently available")
|
||||
raise ValueError("Unrecognised sms provider {}".format(identifier))
|
||||
if identifier == 'firetext':
|
||||
return 'mmg'
|
||||
elif identifier == 'mmg':
|
||||
return 'firetext'
|
||||
raise ValueError('Unrecognised sms provider {}'.format(identifier))
|
||||
|
||||
|
||||
def dao_get_provider_versions(provider_id):
|
||||
stmt = (
|
||||
select(ProviderDetailsHistory)
|
||||
.where(ProviderDetailsHistory.id == provider_id)
|
||||
.order_by(desc(ProviderDetailsHistory.version))
|
||||
.limit(100)
|
||||
return ProviderDetailsHistory.query.filter_by(
|
||||
id=provider_id
|
||||
).order_by(
|
||||
desc(ProviderDetailsHistory.version)
|
||||
).all()
|
||||
|
||||
|
||||
def _adjust_provider_priority(provider, new_priority):
|
||||
current_app.logger.info(
|
||||
f'Adjusting provider priority - {provider.identifier} going from {provider.priority} to {new_priority}'
|
||||
)
|
||||
# limit results instead of adding pagination
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
provider.priority = new_priority
|
||||
|
||||
# Automatic update so set as notify user
|
||||
provider.created_by_id = current_app.config['NOTIFY_USER_ID']
|
||||
|
||||
# update without commit so that both rows can be changed without ending the transaction
|
||||
# and releasing the for_update lock
|
||||
_update_provider_details_without_commit(provider)
|
||||
|
||||
|
||||
def _get_sms_providers_for_update(time_threshold):
|
||||
@@ -45,42 +56,80 @@ def _get_sms_providers_for_update(time_threshold):
|
||||
release the transaction in that case
|
||||
"""
|
||||
# get current priority of both providers
|
||||
stmt = (
|
||||
select(ProviderDetails)
|
||||
.where(
|
||||
ProviderDetails.notification_type == NotificationType.SMS,
|
||||
ProviderDetails.active,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
q = db.session.execute(stmt).scalars().all()
|
||||
q = ProviderDetails.query.filter(
|
||||
ProviderDetails.notification_type == 'sms',
|
||||
ProviderDetails.active
|
||||
).with_for_update().all()
|
||||
|
||||
# if something updated recently, don't update again. If the updated_at is null, treat it as min time
|
||||
if any(
|
||||
(provider.updated_at or datetime.min) > utc_now() - time_threshold
|
||||
for provider in q
|
||||
):
|
||||
current_app.logger.info(
|
||||
f"Not adjusting providers, providers updated less than {time_threshold} ago."
|
||||
)
|
||||
if any((provider.updated_at or datetime.min) > datetime.utcnow() - time_threshold for provider in q):
|
||||
current_app.logger.info(f"Not adjusting providers, providers updated less than {time_threshold} ago.")
|
||||
return []
|
||||
|
||||
return q
|
||||
|
||||
|
||||
def get_provider_details_by_notification_type(
|
||||
notification_type, supports_international=False
|
||||
):
|
||||
@transactional
|
||||
def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
|
||||
"""
|
||||
Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
|
||||
If either provider has been updated in the last `time_threshold`, then it won't take any action.
|
||||
"""
|
||||
amount_to_reduce_by = 10
|
||||
providers_list = _get_sms_providers_for_update(time_threshold)
|
||||
|
||||
if not providers_list:
|
||||
return
|
||||
|
||||
providers = {provider.identifier: provider for provider in providers_list}
|
||||
other_identifier = get_alternative_sms_provider(identifier)
|
||||
|
||||
reduced_provider = providers[identifier]
|
||||
increased_provider = providers[other_identifier]
|
||||
|
||||
# always keep values between 0 and 100
|
||||
reduced_provider_priority = max(0, reduced_provider.priority - amount_to_reduce_by)
|
||||
increased_provider_priority = min(100, increased_provider.priority + amount_to_reduce_by)
|
||||
|
||||
_adjust_provider_priority(reduced_provider, reduced_provider_priority)
|
||||
_adjust_provider_priority(increased_provider, increased_provider_priority)
|
||||
|
||||
|
||||
@transactional
|
||||
def dao_adjust_provider_priority_back_to_resting_points():
|
||||
"""
|
||||
Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
|
||||
each towards their defined resting points (set in SMS_PROVIDER_RESTING_POINTS in config.py).
|
||||
"""
|
||||
amount_to_reduce_by = 10
|
||||
time_threshold = timedelta(hours=1)
|
||||
|
||||
providers = _get_sms_providers_for_update(time_threshold)
|
||||
|
||||
for provider in providers:
|
||||
target = current_app.config['SMS_PROVIDER_RESTING_POINTS'][provider.identifier]
|
||||
current = provider.priority
|
||||
|
||||
if current != target:
|
||||
if current > target:
|
||||
new_priority = max(target, provider.priority - amount_to_reduce_by)
|
||||
else:
|
||||
new_priority = min(target, provider.priority + amount_to_reduce_by)
|
||||
|
||||
_adjust_provider_priority(provider, new_priority)
|
||||
|
||||
|
||||
def get_provider_details_by_notification_type(notification_type, supports_international=False):
|
||||
|
||||
filters = [ProviderDetails.notification_type == notification_type]
|
||||
|
||||
if supports_international:
|
||||
filters.append(ProviderDetails.supports_international == supports_international)
|
||||
|
||||
stmt = select(ProviderDetails).where(*filters)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def dao_update_provider_details(provider_details):
|
||||
_update_provider_details_without_commit(provider_details)
|
||||
|
||||
@@ -90,7 +139,7 @@ def _update_provider_details_without_commit(provider_details):
|
||||
Doesn't commit, for when you need to control the database transaction manually
|
||||
"""
|
||||
provider_details.version += 1
|
||||
provider_details.updated_at = utc_now()
|
||||
provider_details.updated_at = datetime.utcnow()
|
||||
history = ProviderDetailsHistory.from_original(provider_details)
|
||||
db.session.add(provider_details)
|
||||
db.session.add(history)
|
||||
@@ -99,44 +148,37 @@ def _update_provider_details_without_commit(provider_details):
|
||||
def dao_get_provider_stats():
|
||||
# this query does not include the current day since the task to populate ft_billing runs overnight
|
||||
|
||||
current_datetime = utc_now()
|
||||
first_day_of_the_month = current_datetime.date().replace(day=1)
|
||||
current_bst_datetime = convert_utc_to_bst(datetime.utcnow())
|
||||
first_day_of_the_month = current_bst_datetime.date().replace(day=1)
|
||||
|
||||
substmt = (
|
||||
db.session.query(
|
||||
FactBilling.provider,
|
||||
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label(
|
||||
"current_month_billable_sms"
|
||||
),
|
||||
)
|
||||
.where(
|
||||
FactBilling.notification_type == NotificationType.SMS,
|
||||
FactBilling.local_date >= first_day_of_the_month,
|
||||
)
|
||||
.group_by(FactBilling.provider)
|
||||
.subquery()
|
||||
)
|
||||
subquery = db.session.query(
|
||||
FactBilling.provider,
|
||||
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('current_month_billable_sms')
|
||||
).filter(
|
||||
FactBilling.notification_type == SMS_TYPE,
|
||||
FactBilling.bst_date >= first_day_of_the_month
|
||||
).group_by(
|
||||
FactBilling.provider
|
||||
).subquery()
|
||||
|
||||
result = (
|
||||
db.session.query(
|
||||
ProviderDetails.id,
|
||||
ProviderDetails.display_name,
|
||||
ProviderDetails.identifier,
|
||||
ProviderDetails.notification_type,
|
||||
ProviderDetails.active,
|
||||
ProviderDetails.updated_at,
|
||||
ProviderDetails.supports_international,
|
||||
User.name.label("created_by_name"),
|
||||
func.coalesce(substmt.c.current_month_billable_sms, 0).label(
|
||||
"current_month_billable_sms"
|
||||
),
|
||||
)
|
||||
.outerjoin(substmt, ProviderDetails.identifier == substmt.c.provider)
|
||||
.outerjoin(User, ProviderDetails.created_by_id == User.id)
|
||||
.order_by(
|
||||
ProviderDetails.notification_type,
|
||||
)
|
||||
.all()
|
||||
)
|
||||
result = db.session.query(
|
||||
ProviderDetails.id,
|
||||
ProviderDetails.display_name,
|
||||
ProviderDetails.identifier,
|
||||
ProviderDetails.priority,
|
||||
ProviderDetails.notification_type,
|
||||
ProviderDetails.active,
|
||||
ProviderDetails.updated_at,
|
||||
ProviderDetails.supports_international,
|
||||
User.name.label('created_by_name'),
|
||||
func.coalesce(subquery.c.current_month_billable_sms, 0).label('current_month_billable_sms')
|
||||
).outerjoin(
|
||||
subquery, ProviderDetails.identifier == subquery.c.provider
|
||||
).outerjoin(
|
||||
User, ProviderDetails.created_by_id == User.id
|
||||
).order_by(
|
||||
ProviderDetails.notification_type,
|
||||
ProviderDetails.priority,
|
||||
).all()
|
||||
|
||||
return result
|
||||
|
||||
11
app/dao/provider_rates_dao.py
Normal file
11
app/dao/provider_rates_dao.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from app.models import ProviderRates, ProviderDetails
|
||||
from app import db
|
||||
from app.dao.dao_utils import transactional
|
||||
|
||||
|
||||
@transactional
|
||||
def create_provider_rates(provider_identifier, valid_from, rate):
|
||||
provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
|
||||
|
||||
provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
|
||||
db.session.add(provider_rates)
|
||||
98
app/dao/returned_letters_dao.py
Normal file
98
app/dao/returned_letters_dao.py
Normal file
@@ -0,0 +1,98 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import func, desc
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import (
|
||||
Job,
|
||||
Notification,
|
||||
NotificationHistory,
|
||||
ReturnedLetter,
|
||||
Template,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def _get_notification_ids_for_references(references):
|
||||
notification_ids = db.session.query(Notification.id, Notification.service_id).filter(
|
||||
Notification.reference.in_(references)
|
||||
).all()
|
||||
|
||||
notification_history_ids = db.session.query(NotificationHistory.id, NotificationHistory.service_id).filter(
|
||||
NotificationHistory.reference.in_(references)
|
||||
).all()
|
||||
|
||||
return notification_ids + notification_history_ids
|
||||
|
||||
|
||||
@transactional
|
||||
def insert_or_update_returned_letters(references):
|
||||
data = _get_notification_ids_for_references(references)
|
||||
for row in data:
|
||||
table = ReturnedLetter.__table__
|
||||
|
||||
stmt = insert(table).values(
|
||||
reported_at=datetime.utcnow().date(),
|
||||
service_id=row.service_id,
|
||||
notification_id=row.id,
|
||||
created_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[table.c.notification_id],
|
||||
set_={
|
||||
'reported_at': datetime.utcnow().date(),
|
||||
'updated_at': datetime.utcnow()
|
||||
}
|
||||
)
|
||||
db.session.connection().execute(stmt)
|
||||
|
||||
|
||||
def fetch_returned_letter_summary(service_id):
|
||||
return db.session.query(
|
||||
func.count(ReturnedLetter.notification_id).label('returned_letter_count'),
|
||||
ReturnedLetter.reported_at
|
||||
).filter(
|
||||
ReturnedLetter.service_id == service_id,
|
||||
).group_by(
|
||||
ReturnedLetter.reported_at
|
||||
).order_by(
|
||||
desc(ReturnedLetter.reported_at)
|
||||
).all()
|
||||
|
||||
|
||||
def fetch_returned_letters(service_id, report_date):
|
||||
results = []
|
||||
for table in [Notification, NotificationHistory]:
|
||||
query = db.session.query(
|
||||
ReturnedLetter.notification_id,
|
||||
ReturnedLetter.reported_at,
|
||||
table.client_reference,
|
||||
table.created_at,
|
||||
Template.name.label('template_name'),
|
||||
table.template_id,
|
||||
table.template_version,
|
||||
Template.hidden,
|
||||
table.api_key_id,
|
||||
table.created_by_id,
|
||||
User.name.label('user_name'),
|
||||
User.email_address,
|
||||
Job.original_file_name,
|
||||
(table.job_row_number + 1).label('job_row_number') # row numbers start at 0
|
||||
).outerjoin(
|
||||
User, table.created_by_id == User.id
|
||||
).outerjoin(
|
||||
Job, table.job_id == Job.id
|
||||
).filter(
|
||||
ReturnedLetter.service_id == service_id,
|
||||
ReturnedLetter.reported_at == report_date,
|
||||
ReturnedLetter.notification_id == table.id,
|
||||
table.template_id == Template.id
|
||||
).order_by(
|
||||
desc(ReturnedLetter.reported_at), desc(table.created_at)
|
||||
)
|
||||
results = results + query.all()
|
||||
results = sorted(results, key=lambda i: i.created_at, reverse=True)
|
||||
return results
|
||||
@@ -1,74 +1,51 @@
|
||||
from sqlalchemy import select
|
||||
from datetime import datetime
|
||||
|
||||
from app import create_uuid, db
|
||||
from app.dao.dao_utils import autocommit, version_class
|
||||
from app.enums import CallbackType
|
||||
from app import db, create_uuid
|
||||
from app.dao.dao_utils import transactional, version_class
|
||||
from app.models import ServiceCallbackApi
|
||||
from app.utils import utc_now
|
||||
|
||||
from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
@version_class(ServiceCallbackApi)
|
||||
def save_service_callback_api(service_callback_api):
|
||||
service_callback_api.id = create_uuid()
|
||||
service_callback_api.created_at = utc_now()
|
||||
service_callback_api.created_at = datetime.utcnow()
|
||||
db.session.add(service_callback_api)
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
@version_class(ServiceCallbackApi)
|
||||
def reset_service_callback_api(
|
||||
service_callback_api, updated_by_id, url=None, bearer_token=None
|
||||
):
|
||||
def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
|
||||
if url:
|
||||
service_callback_api.url = url
|
||||
if bearer_token:
|
||||
service_callback_api.bearer_token = bearer_token
|
||||
service_callback_api.updated_by_id = updated_by_id
|
||||
service_callback_api.updated_at = utc_now()
|
||||
service_callback_api.updated_at = datetime.utcnow()
|
||||
|
||||
db.session.add(service_callback_api)
|
||||
|
||||
|
||||
def get_service_callback_api(service_callback_api_id, service_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(ServiceCallbackApi).where(
|
||||
ServiceCallbackApi.id == service_callback_api_id,
|
||||
ServiceCallbackApi.service_id == service_id,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
return ServiceCallbackApi.query.filter_by(id=service_callback_api_id, service_id=service_id).first()
|
||||
|
||||
|
||||
def get_service_delivery_status_callback_api_for_service(service_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(ServiceCallbackApi).where(
|
||||
ServiceCallbackApi.service_id == service_id,
|
||||
ServiceCallbackApi.callback_type == CallbackType.DELIVERY_STATUS,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
return ServiceCallbackApi.query.filter_by(
|
||||
service_id=service_id,
|
||||
callback_type=DELIVERY_STATUS_CALLBACK_TYPE
|
||||
).first()
|
||||
|
||||
|
||||
def get_service_complaint_callback_api_for_service(service_id):
|
||||
return (
|
||||
db.session.execute(
|
||||
select(ServiceCallbackApi).where(
|
||||
ServiceCallbackApi.service_id == service_id,
|
||||
ServiceCallbackApi.callback_type == CallbackType.COMPLAINT,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
return ServiceCallbackApi.query.filter_by(
|
||||
service_id=service_id,
|
||||
callback_type=COMPLAINT_CALLBACK_TYPE
|
||||
).first()
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def delete_service_callback_api(service_callback_api):
|
||||
db.session.delete(service_callback_api)
|
||||
|
||||
@@ -1,71 +1,52 @@
|
||||
from sqlalchemy import select, update
|
||||
from datetime import datetime
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.models import ServiceDataRetention
|
||||
from app.utils import utc_now
|
||||
|
||||
|
||||
def fetch_service_data_retention_by_id(service_id, data_retention_id):
|
||||
stmt = select(ServiceDataRetention).where(
|
||||
ServiceDataRetention.service_id == service_id,
|
||||
ServiceDataRetention.id == data_retention_id,
|
||||
)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
data_retention = ServiceDataRetention.query.filter_by(service_id=service_id, id=data_retention_id).first()
|
||||
return data_retention
|
||||
|
||||
|
||||
def fetch_service_data_retention(service_id):
|
||||
stmt = (
|
||||
select(ServiceDataRetention)
|
||||
.where(ServiceDataRetention.service_id == service_id)
|
||||
.order_by(
|
||||
# in the order that models.notification_types are created (email, sms, letter)
|
||||
ServiceDataRetention.notification_type
|
||||
)
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
data_retention_list = ServiceDataRetention.query.filter_by(
|
||||
service_id=service_id
|
||||
).order_by(
|
||||
# in the order that models.notification_types are created (email, sms, letter)
|
||||
ServiceDataRetention.notification_type
|
||||
).all()
|
||||
return data_retention_list
|
||||
|
||||
|
||||
def fetch_service_data_retention_by_notification_type(service_id, notification_type):
|
||||
stmt = select(ServiceDataRetention).where(
|
||||
ServiceDataRetention.service_id == service_id,
|
||||
ServiceDataRetention.notification_type == notification_type,
|
||||
)
|
||||
return db.session.execute(stmt).scalars().first()
|
||||
|
||||
|
||||
@autocommit
|
||||
def insert_service_data_retention(service_id, notification_type, days_of_retention):
|
||||
new_data_retention = ServiceDataRetention(
|
||||
data_retention_list = ServiceDataRetention.query.filter_by(
|
||||
service_id=service_id,
|
||||
notification_type=notification_type,
|
||||
days_of_retention=days_of_retention,
|
||||
)
|
||||
notification_type=notification_type
|
||||
).first()
|
||||
return data_retention_list
|
||||
|
||||
|
||||
@transactional
|
||||
def insert_service_data_retention(service_id, notification_type, days_of_retention):
|
||||
new_data_retention = ServiceDataRetention(service_id=service_id,
|
||||
notification_type=notification_type,
|
||||
days_of_retention=days_of_retention)
|
||||
|
||||
db.session.add(new_data_retention)
|
||||
return new_data_retention
|
||||
|
||||
|
||||
@autocommit
|
||||
def update_service_data_retention(
|
||||
service_data_retention_id, service_id, days_of_retention
|
||||
):
|
||||
stmt = (
|
||||
update(ServiceDataRetention)
|
||||
.where(
|
||||
ServiceDataRetention.id == service_data_retention_id,
|
||||
ServiceDataRetention.service_id == service_id,
|
||||
)
|
||||
.values({"days_of_retention": days_of_retention, "updated_at": utc_now()})
|
||||
@transactional
|
||||
def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
|
||||
updated_count = ServiceDataRetention.query.filter(
|
||||
ServiceDataRetention.id == service_data_retention_id,
|
||||
ServiceDataRetention.service_id == service_id
|
||||
).update(
|
||||
{
|
||||
"days_of_retention": days_of_retention,
|
||||
"updated_at": datetime.utcnow()
|
||||
}
|
||||
)
|
||||
result = db.session.execute(stmt)
|
||||
return result.rowcount
|
||||
|
||||
|
||||
def fetch_service_data_retention_for_all_services_by_notification_type(
|
||||
notification_type,
|
||||
):
|
||||
stmt = select(ServiceDataRetention).where(
|
||||
ServiceDataRetention.notification_type == notification_type
|
||||
)
|
||||
return db.session.execute(stmt).scalars().all()
|
||||
return updated_count
|
||||
|
||||
@@ -1,42 +1,34 @@
|
||||
from sqlalchemy import desc, select
|
||||
from sqlalchemy import desc
|
||||
|
||||
from app import db
|
||||
from app.dao.dao_utils import autocommit
|
||||
from app.dao.dao_utils import transactional
|
||||
from app.errors import InvalidRequest
|
||||
from app.exceptions import ArchiveValidationError
|
||||
from app.models import ServiceEmailReplyTo
|
||||
|
||||
|
||||
def dao_get_reply_to_by_service_id(service_id):
|
||||
reply_to = (
|
||||
db.session.query(ServiceEmailReplyTo)
|
||||
.where(
|
||||
ServiceEmailReplyTo.service_id == service_id,
|
||||
ServiceEmailReplyTo.archived == False, # noqa
|
||||
)
|
||||
.order_by(
|
||||
desc(ServiceEmailReplyTo.is_default), desc(ServiceEmailReplyTo.created_at)
|
||||
)
|
||||
.all()
|
||||
)
|
||||
reply_to = db.session.query(
|
||||
ServiceEmailReplyTo
|
||||
).filter(
|
||||
ServiceEmailReplyTo.service_id == service_id,
|
||||
ServiceEmailReplyTo.archived == False # noqa
|
||||
).order_by(desc(ServiceEmailReplyTo.is_default), desc(ServiceEmailReplyTo.created_at)).all()
|
||||
return reply_to
|
||||
|
||||
|
||||
def dao_get_reply_to_by_id(service_id, reply_to_id):
|
||||
reply_to = (
|
||||
db.session.query(ServiceEmailReplyTo)
|
||||
.where(
|
||||
ServiceEmailReplyTo.service_id == service_id,
|
||||
ServiceEmailReplyTo.id == reply_to_id,
|
||||
ServiceEmailReplyTo.archived == False, # noqa
|
||||
)
|
||||
.order_by(ServiceEmailReplyTo.created_at)
|
||||
.one()
|
||||
)
|
||||
reply_to = db.session.query(
|
||||
ServiceEmailReplyTo
|
||||
).filter(
|
||||
ServiceEmailReplyTo.service_id == service_id,
|
||||
ServiceEmailReplyTo.id == reply_to_id,
|
||||
ServiceEmailReplyTo.archived == False # noqa
|
||||
).order_by(ServiceEmailReplyTo.created_at).one()
|
||||
return reply_to
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def add_reply_to_email_address_for_service(service_id, email_address, is_default):
|
||||
old_default = _get_existing_default(service_id)
|
||||
if is_default:
|
||||
@@ -44,48 +36,36 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
|
||||
else:
|
||||
_raise_when_no_default(old_default)
|
||||
|
||||
new_reply_to = ServiceEmailReplyTo(
|
||||
service_id=service_id, email_address=email_address, is_default=is_default
|
||||
)
|
||||
new_reply_to = ServiceEmailReplyTo(service_id=service_id, email_address=email_address, is_default=is_default)
|
||||
db.session.add(new_reply_to)
|
||||
return new_reply_to
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
|
||||
old_default = _get_existing_default(service_id)
|
||||
if is_default:
|
||||
_reset_old_default_to_false(old_default)
|
||||
else:
|
||||
if old_default.id == reply_to_id:
|
||||
raise InvalidRequest(
|
||||
"You must have at least one reply to email address as the default.", 400
|
||||
)
|
||||
raise InvalidRequest("You must have at least one reply to email address as the default.", 400)
|
||||
|
||||
reply_to_update = db.session.get(ServiceEmailReplyTo, reply_to_id)
|
||||
reply_to_update = ServiceEmailReplyTo.query.get(reply_to_id)
|
||||
reply_to_update.email_address = email_address
|
||||
reply_to_update.is_default = is_default
|
||||
db.session.add(reply_to_update)
|
||||
return reply_to_update
|
||||
|
||||
|
||||
@autocommit
|
||||
@transactional
|
||||
def archive_reply_to_email_address(service_id, reply_to_id):
|
||||
reply_to_archive = (
|
||||
db.session.execute(
|
||||
select(ServiceEmailReplyTo).where(
|
||||
ServiceEmailReplyTo.id == reply_to_id,
|
||||
ServiceEmailReplyTo.service_id == service_id,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.one()
|
||||
)
|
||||
reply_to_archive = ServiceEmailReplyTo.query.filter_by(
|
||||
id=reply_to_id,
|
||||
service_id=service_id
|
||||
).one()
|
||||
|
||||
if reply_to_archive.is_default:
|
||||
raise ArchiveValidationError(
|
||||
"You cannot delete a default email reply to address"
|
||||
)
|
||||
raise ArchiveValidationError("You cannot delete a default email reply to address")
|
||||
|
||||
reply_to_archive.archived = True
|
||||
|
||||
@@ -102,9 +82,7 @@ def _get_existing_default(service_id):
|
||||
else:
|
||||
raise Exception(
|
||||
"There should only be one default reply to email for each service. Service {} has {}".format(
|
||||
service_id, len(old_default)
|
||||
)
|
||||
)
|
||||
service_id, len(old_default)))
|
||||
return None
|
||||
|
||||
|
||||
@@ -117,6 +95,4 @@ def _reset_old_default_to_false(old_default):
|
||||
def _raise_when_no_default(old_default):
|
||||
# check that the update is not updating the only default to false
|
||||
if not old_default:
|
||||
raise InvalidRequest(
|
||||
"You must have at least one reply to email address as the default.", 400
|
||||
)
|
||||
raise InvalidRequest("You must have at least one reply to email address as the default.", 400)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user