Added a test

By added the notification_type to the filter we get a better performance on the query. Especially when selecting for NotificationHistory.

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
+# W503: line break before binary operator
+exclude = venv*,__pycache__,node_modules,cache,migrations,build
+ignore = W503
+max-complexity = 14
+max-line-length = 120

.github/workflows/checks.yml

@@ -1,76 +0,0 @@
-name: Run checks
-
-on: [push]
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  ANTIVIRUS_ENABLED: 0
-  NOTIFY_ENVIRONMENT: development
-  NOTIFICATION_QUEUE_PREFIX: local_dev_10x
-  STATSD_HOST: localhost
-  SES_STUB_URL: None
-  NOTIFY_APP_NAME: api
-  NOTIFY_EMAIL_DOMAIN: dispostable.com
-  NOTIFY_LOG_PATH: /workspace/logs/app.log
-  ADMIN_CLIENT_ID: notify-admin
-  ADMIN_CLIENT_SECRET: dev-notify-secret-key
-  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
-  FLASK_APP: application.py
-  FLASK_ENV: development
-  WERKZEUG_DEBUG_PIN: off
-  ADMIN_BASE_URL: http://localhost:6012
-  API_HOST_NAME: http://localhost:6011
-  REDIS_URL: redis://localhost:6380
-  REDIS_ENABLED: False
-  SQLALCHEMY_DATABASE_URI: postgresql://postgres:chummy@db:5432/notification_api
-  SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-  AWS_REGION: us-west-2
-  AWS_PINPOINT_REGION: us-west-2
-  AWS_US_TOLL_FREE_NUMBER: +18446120782
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
-    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-    - name: Install application dependencies
-      run: make bootstrap
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-    # - name: Run style checks
-    #   run: flake8 .
-    # - name: Check imports alphabetized
-    #   run: isort --check-only ./app ./tests
-    - name: Run tests
-      run: pytest -n4 --maxfail=10
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api

.github/workflows/deploy.yml

@@ -1,58 +0,0 @@
-name: Deploy to prototype environment
-
-on:
-  workflow_run:
-    workflows: [ Run checks ]
-    types:
-      - completed
-    branches: [ main ] # Redundant, workflow_run events are only triggered on default branch (`main`)
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-
-    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-
-    - uses: actions/checkout@v3
-
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-
-    - name: Install application dependencies
-      run: make bootstrap
-
-    - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
-      env:
-        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      with:
-        cf_username: ${{ secrets.cloudgov_username }}
-        cf_password: ${{ secrets.cloudgov_password }}
-        cf_org: gsa-10x-prototyping
-        cf_space: 10x-notifications
-        full_command: |
-          cf push --strategy rolling \
-          --var DANGEROUS_SALT="$DANGEROUS_SALT" \
-          --var SECRET_KEY="$SECRET_KEY" \
-          --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-          --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
-
-  bail:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - run: echo 'Checks failed, not deploying'

.gitignore

@@ -69,15 +69,10 @@ target/
 *.DS_Store
 environment.sh
 .envrc
-.env
-.env*
-varsfile
 
 celerybeat-schedule
 
 # CloudFoundry
 .cf
-varsfile*
-.secret*
 
 /scripts/run_my_tests.sh

.pyup.yml

@@ -4,5 +4,5 @@ schedule: "every week on wednesday"
 
 search: False
 requirements:
-  - requirements.in
+  - requirements-app.txt
-  - requirements_for_test.txt
+  - requirements-dev.txt

Makefile

@@ -7,57 +7,28 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
+
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
+
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
+
 CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
-CF_MANIFEST_PATH ?= /tmp/manifest.yml
+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
-
 
 NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
 
 ## DEVELOPMENT
 
-.PHONY: bootstrap
-bootstrap: generate-version-file ## Set up everything to run the app
-	pip3 install -r requirements_for_test.txt
-	createdb notification_api || true
-	(flask db upgrade) || true
-
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
-
-.PHONY: run-flask
-run-flask: ## Run flask
-	flask run -p 6011 --host=0.0.0.0
-
-.PHONY: run-celery
-run-celery: ## Run celery, TODO remove purge for staging/prod
-	celery -A run_celery.notify_celery purge -f
-	celery \
-		-A run_celery.notify_celery worker \
-		--pidfile="/tmp/celery.pid" \
-		--loglevel=INFO \
-		--concurrency=4
-
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
-.PHONY: run-celery-beat
-run-celery-beat: ## Run celery beat
-	celery \
-	-A run_celery.notify_celery beat \
-	--loglevel=INFO
-
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -67,19 +38,71 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: test
-test: ## Run tests
+test: generate-version-file ## Run tests
-	# flake8 .
+	./scripts/run_tests.sh
-	isort --check-only ./app ./tests
-	pytest -n4 --maxfail=10
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pip install --upgrade pip-tools
+	rm -rf venv-freeze
-	pip-compile requirements.in
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
+
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
+
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
+	docker build -f docker/Dockerfile \
+		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
+		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
+		--build-arg NO_PROXY="${NO_PROXY}" \
+		-t ${DOCKER_BUILDER_IMAGE_NAME} \
+		.
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	@docker run -it --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
+		-e GIT_COMMIT=${GIT_COMMIT} \
+		-e BUILD_NUMBER=${BUILD_NUMBER} \
+		-e BUILD_URL=${BUILD_URL} \
+		-e http_proxy="${HTTP_PROXY}" \
+		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e https_proxy="${HTTPS_PROXY}" \
+		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e NO_PROXY="${NO_PROXY}" \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: create-docker-test-db
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	@docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
 
 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
+	rm -rf node_modules cache target venv .coverage build tests/.cache
 
 
 ## DEPLOYMENT
@@ -131,38 +154,34 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 	@cf app --guid ${CF_APP} || exit 1
 
 	# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
-	cf cancel-deployment ${CF_APP} || true
+	cf v3-cancel-zdt-push ${CF_APP} || true
+
+	cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
+	CF_STARTUP_TIMEOUT=15 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete  # fails after 15 mins if deploy doesn't work
 
-	# fails after 15 mins if deploy doesn't work
-	CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling
 
 .PHONY: cf-deploy-api-db-migration
 cf-deploy-api-db-migration:
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
-	make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH}
+	cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
-
+	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
-	cf push notifications-api --no-route -f ${CF_MANIFEST_PATH}
-	rm ${CF_MANIFEST_PATH}
-
-	cf run-task notifications-api --command="flask db upgrade" --name api_db_migration
 
 .PHONY: cf-check-api-db-migration-task
-cf-check-api-db-migration-task: ## Get the status for the last notifications-api task
+cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
-	@cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state"
+	@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"
 
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
 	$(if ${CF_APP},,$(error Must specify CF_APP))
-	rm ${CF_MANIFEST_PATH}
+	cf v3-cancel-zdt-push ${CF_APP}
-	cf cancel-deployment ${CF_APP}
 
 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:
 	@echo $(shell python3 scripts/check_if_new_migration.py)
 
 .PHONY: cf-deploy-failwhale
-cf-deploy-failwhale:
+cf-deploy-failwhale:  #
 	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
 	cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
 

Procfile

@@ -1,2 +1 @@
-web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
+web: ./scripts/paas_app_wrapper.sh
-worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null

README.md

@@ -1,142 +1,158 @@
-# US Notify API
+# GOV.UK Notify API
-
-Cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
 
 Contains:
-
+- the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation)
-- the public-facing REST API for US Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation) [DOCS ARE STILL UK]
+- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to)
-- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/18F/notifications-admin) talks to)
 - asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc
 
-## QUICKSTART
----
-If you are the first on your team to deploy, set up AWS SES/SNS as instructed in the AWS setup section below.
-
-Create .env file as described in the .env section below.
-
-Install VS Code
-Open VS Code and install the Remote-Containers plug-in from Microsoft.
-
-Make sure your docker daemon is running (on OS X, this is typically accomplished by opening the Docker Desktop app)
-
-Create the external docker network:
-
-`docker network create notify-network`
-
-Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...”
-When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). 
-
-After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes.
-
-Select View->Open View..., then search/select “ports”. Await a green dot on the port view, then open a new terminal and run the web server:
-`make run-flask`
-
-Open another terminal and run the background tasks:
-`make run-celery`
-
----
 ## Setting Up
 
-### `.env` file
+### Python version
 
-Create and edit a .env file, based on sample.env. 
+At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.
 
-NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
+### AWS credentials
 
-Things to change:
+To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.
 
-- If you're not the first to deploy, only replace the aws creds, get these from team lead
+Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
-- Replace `NOTIFICATION_QUEUE_PREFIX` with `local_dev_<your org>_`
-- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project)
-- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values
-- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars
 
-### AWS Setup
+### Virtualenv
-
-**Steps to prepare SES**
-
-1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
-2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
-
-TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures
-
-**Steps to prepare SNS**
-
-1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms"
-2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save
-3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers"
-4. Under "Number Settings" choose "Request Phone Number"
-5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request"
-6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)"
-7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number)
-
-At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
-
-### Secrets Detection
 
 ```
-brew install detect-secrets # or pip install detect-secrets
+mkvirtualenv -p /usr/local/bin/python3 notifications-api
-detect-secrets scan
-#review output of above, make sure none of the baseline entries are sensitive
-detect-secrets scan > .secrets.baseline
-#creates the baseline file
 ```
 
-Ideally, you'll install `detect-secrets` so that it's accessible from any environment from which you _might_ commit. You can use `brew install` to make it available globally. You could also install via `pip install` inside a virtual environment, if you're sure you'll _only_ commit from that environment.
+### `environment.sh`
 
-If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test).
+Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
 
-You can permit exceptions by adding an inline comment containing `pragma: allowlist secret`
+Create a local environment.sh file containing the following:
 
-The command that is actually run by the pre-commit hook is: `git diff --staged --name-only -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+```
+echo "
+export NOTIFY_ENVIRONMENT='development'
 
-You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+export MMG_API_KEY='MMG_API_KEY'
+export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
+export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
+
+export FLASK_APP=application.py
+export FLASK_DEBUG=1
+export WERKZEUG_DEBUG_PIN=off
+"> environment.sh
+```
+
+NOTES:
+
+ * Replace the placeholder key and prefix values as appropriate
+ * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
+ * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.
 
 ### Postgres
 
-Local postgres implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
+
+Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.
 
 ### Redis
 
-Local redis implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
 
-## To test the application
+        REDIS_ENABLED = True
+
+
+##  To run the application
+
+First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
+
+You need to run the api application and a local celery instance.
+
+There are two run scripts for running all the necessary parts.
 
 ```
-# install dependencies, etc.
+scripts/run_app.sh
-make bootstrap
+```
 
+```
+scripts/run_celery.sh
+```
+
+Optionally you can also run this script to run the scheduled tasks:
+
+```
+scripts/run_celery_beat.sh
+```
+
+
+##  To test the application
+
+First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
+
+Then simply run
+
+```
 make test
 ```
 
-## To run scheduled tasks
+That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
+[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
+
+
+## To update application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with
 
 ```
-# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat 
+make freeze-requirements
-make run-celery-beat
 ```
 
-## To run one off tasks (Ignore for Quick Start)
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.
+
+
+## To run one off tasks
 
 Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
 care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
 example, to purge all dynamically generated functional test data, do the following:
 
-Local (from inside the devcontainer)
+Locally
-
 ```
 flask command purge_functional_test_data -u <functional tests user name prefix>
 ```
 
-Remote
+On the server
-
 ```
 cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
 ```
 
 All commands and command options have a --help command if you need more information.
 
-## Further documentation [DEPRECATED]
 
-- [Writing public APIs](docs/writing-public-apis.md)
+## To create a new worker app
-- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
+
+You need to:
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
+1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
+1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
+1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
+
+**Important:**
+
+Before pushing the deployment change on jenkins, read below about the first time deployment.
+
+### First time deployment of your new worker
+
+Our deployment flow requires that the app is present in order to proceed with the deployment.
+
+This means that the first deployment of your app must happen manually.
+
+To do this:
+
+1. Ensure your code is backwards compatible
+1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
+
+Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.

app/__init__.py

@@ -1,40 +1,38 @@
+import time
 import os
 import random
 import string
-import time
 import uuid
-from time import monotonic
 
 from celery import current_task
-from flask import (
+from flask import _request_ctx_stack, request, g, jsonify, make_response, current_app, has_request_context
-    current_app,
+from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
-    g,
-    has_request_context,
-    jsonify,
-    make_response,
-    request,
-)
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from gds_metrics import GDSMetrics
 from gds_metrics.metrics import Gauge, Histogram
-from notifications_utils import logging, request_helper
+from time import monotonic
-from notifications_utils.celery import NotifyCelery
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
+from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils import logging, request_helper
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy
 
-from app.clients import NotificationProviderClients
+from app.celery.celery import NotifyCelery
-from app.clients.cbc_proxy import CBCProxyClient
+from app.clients import Clients
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.aws_sns import AwsSnsClient
+from app.clients.sms.firetext import FiretextClient
+from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+
+DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+DATE_FORMAT = "%Y-%m-%d"
 
 
 class SQLAlchemy(_SQLAlchemy):
@@ -53,21 +51,22 @@ db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
+firetext_client = FiretextClient()
+mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 aws_ses_stub_client = AwsSesStubClient()
-aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
-cbc_proxy_client = CBCProxyClient()
+performance_platform_client = PerformancePlatformClient()
 document_download_client = DocumentDownloadClient()
 metrics = GDSMetrics()
 
-notification_provider_clients = NotificationProviderClients()
+clients = Clients()
 
-api_user = LocalProxy(lambda: g.api_user)
+api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
-authenticated_service = LocalProxy(lambda: g.authenticated_service)
+authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
 
 CONCURRENT_REQUESTS = Gauge(
     'concurrent_web_request_count',
@@ -84,9 +83,6 @@ def create_app(application):
 
     application.config['NOTIFY_APP_NAME'] = application.name
     init_app(application)
-
-    # Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded
-    metrics.init_app(application)
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)
@@ -94,7 +90,8 @@ def create_app(application):
     zendesk_client.init_app(application)
     statsd_client.init_app(application)
     logging.init_app(application, statsd_client)
-    aws_sns_client.init_app(application, statsd_client=statsd_client)
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application, statsd_client=statsd_client)
 
     aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     aws_ses_stub_client.init_app(
@@ -104,17 +101,14 @@ def create_app(application):
     )
     # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
     email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
-    notification_provider_clients.init_app(
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients)
-        sms_clients=[aws_sns_client],
-        email_clients=email_clients
-    )
 
     notify_celery.init_app(application)
     encryption.init_app(application)
     redis_store.init_app(application)
+    performance_platform_client.init_app(application)
     document_download_client.init_app(application)
-
+    metrics.init_app(application)
-    cbc_proxy_client.init_app(application)
 
     register_blueprint(application)
     register_v2_blueprints(application)
@@ -130,56 +124,34 @@ def create_app(application):
 
 
 def register_blueprint(application):
-    from app.authentication.auth import (
+    from app.service.rest import service_blueprint
-        requires_admin_auth,
+    from app.service.callback_rest import service_callback_blueprint
-        requires_auth,
+    from app.user.rest import user_blueprint
-        requires_govuk_alerts_auth,
+    from app.template.rest import template_blueprint
-        requires_no_auth,
+    from app.status.healthcheck import status as status_blueprint
-    )
+    from app.job.rest import job_blueprint
-    from app.billing.rest import billing_blueprint
+    from app.notifications.rest import notifications as notifications_blueprint
-    from app.broadcast_message.rest import broadcast_message_blueprint
+    from app.invite.rest import invite as invite_blueprint
-    from app.complaint.complaint_rest import complaint_blueprint
+    from app.accept_invite.rest import accept_invite
-    from app.email_branding.rest import email_branding_blueprint
+    from app.template_statistics.rest import template_statistics as template_statistics_blueprint
     from app.events.rest import events as events_blueprint
-    from app.govuk_alerts.rest import govuk_alerts_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.email_branding.rest import email_branding_blueprint
     from app.inbound_number.rest import inbound_number_blueprint
     from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
-    from app.job.rest import job_blueprint
+    from app.notifications.receive_notifications import receive_notifications_blueprint
-    from app.letter_branding.letter_branding_rest import (
+    from app.notifications.notifications_sms_callback import sms_callback_blueprint
-        letter_branding_blueprint,
+    from app.notifications.notifications_letter_callback import letter_callback_blueprint
-    )
+    from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
     from app.letters.rest import letter_job
-    from app.notifications.notifications_letter_callback import (
+    from app.billing.rest import billing_blueprint
-        letter_callback_blueprint,
-    )
-    from app.notifications.notifications_sms_callback import (
-        sms_callback_blueprint,
-    )
-    from app.notifications.receive_notifications import (
-        receive_notifications_blueprint,
-    )
-    from app.notifications.rest import notifications as notifications_blueprint
-    from app.organisation.invite_rest import organisation_invite_blueprint
     from app.organisation.rest import organisation_blueprint
-    from app.performance_dashboard.rest import performance_dashboard_blueprint
+    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.complaint.complaint_rest import complaint_blueprint
     from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import (
-        provider_details as provider_details_blueprint,
-    )
-    from app.service.callback_rest import service_callback_blueprint
-    from app.service.rest import service_blueprint
-    from app.service_invite.rest import (
-        service_invite as service_invite_blueprint,
-    )
-    from app.status.healthcheck import status as status_blueprint
-    from app.template.rest import template_blueprint
     from app.template_folder.rest import template_folder_blueprint
-    from app.template_statistics.rest import (
+    from app.letter_branding.letter_branding_rest import letter_branding_blueprint
-        template_statistics as template_statistics_blueprint,
-    )
     from app.upload.rest import upload_blueprint
-    from app.user.rest import user_blueprint
-    from app.webauthn.rest import webauthn_blueprint
 
     service_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -187,9 +159,6 @@ def register_blueprint(application):
     user_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(user_blueprint, url_prefix='/user')
 
-    webauthn_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(webauthn_blueprint)
-
     template_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_blueprint)
 
@@ -211,11 +180,8 @@ def register_blueprint(application):
     job_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(job_blueprint)
 
-    service_invite_blueprint.before_request(requires_admin_auth)
+    invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_invite_blueprint)
+    application.register_blueprint(invite_blueprint)
-
-    organisation_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organisation_invite_blueprint)
 
     inbound_number_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_number_blueprint)
@@ -223,6 +189,9 @@ def register_blueprint(application):
     inbound_sms_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_sms_blueprint)
 
+    accept_invite.before_request(requires_admin_auth)
+    application.register_blueprint(accept_invite, url_prefix='/invite')
+
     template_statistics_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_statistics_blueprint)
 
@@ -250,12 +219,12 @@ def register_blueprint(application):
     organisation_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(organisation_blueprint, url_prefix='/organisations')
 
+    organisation_invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(organisation_invite_blueprint)
+
     complaint_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(complaint_blueprint)
 
-    performance_dashboard_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(performance_dashboard_blueprint)
-
     platform_stats_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')
 
@@ -268,47 +237,41 @@ def register_blueprint(application):
     upload_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(upload_blueprint)
 
-    broadcast_message_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(broadcast_message_blueprint)
-
-    govuk_alerts_blueprint.before_request(requires_govuk_alerts_auth)
-    application.register_blueprint(govuk_alerts_blueprint)
-
 
 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
+    from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
+    from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
+    from app.v2.template.get_template import v2_template_blueprint as get_template
+    from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
+    from app.v2.template.post_template import v2_template_blueprint as post_template
     from app.authentication.auth import requires_auth
-    from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint
 
-    v2_notification_blueprint.before_request(requires_auth)
+    post_notifications.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
+    application.register_blueprint(post_notifications)
 
-    v2_templates_blueprint.before_request(requires_auth)
+    get_notifications.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
+    application.register_blueprint(get_notifications)
 
-    v2_template_blueprint.before_request(requires_auth)
+    get_templates.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
+    application.register_blueprint(get_templates)
 
-    v2_inbound_sms_blueprint.before_request(requires_auth)
+    get_template.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
+    application.register_blueprint(get_template)
 
-    v2_broadcast_blueprint.before_request(requires_auth)
+    post_template.before_request(requires_auth)
-    application.register_blueprint(v2_broadcast_blueprint)
+    application.register_blueprint(post_template)
+
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)
 
 
 def init_app(app):
 
+    @app.before_request
+    def record_user_agent():
+        statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))
+
     @app.before_request
     def record_request_details():
         CONCURRENT_REQUESTS.inc()
@@ -355,6 +318,18 @@ def create_random_identifier():
     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
 
 
+def process_user_agent(user_agent_string):
+    if user_agent_string and user_agent_string.lower().startswith("notify"):
+        components = user_agent_string.split("/")
+        client_name = components[0].lower()
+        client_version = components[1].replace(".", "-")
+        return "{}.{}".format(client_name, client_version)
+    elif user_agent_string and not user_agent_string.lower().startswith("notify"):
+        return "non-notify-user-agent"
+    else:
+        return "unknown"
+
+
 def setup_sqlalchemy_events(app):
 
     TOTAL_DB_CONNECTIONS = Gauge(
@@ -387,55 +362,49 @@ def setup_sqlalchemy_events(app):
 
         @event.listens_for(db.engine, 'checkout')
         def checkout(dbapi_connection, connection_record, connection_proxy):
-            try:
+            # connection given to a web worker
-                # connection given to a web worker
+            TOTAL_CHECKED_OUT_DB_CONNECTIONS.inc()
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.inc()
 
-                # this will overwrite any previous checkout_at timestamp
+            # this will overwrite any previous checkout_at timestamp
-                connection_record.info['checkout_at'] = time.monotonic()
+            connection_record.info['checkout_at'] = time.monotonic()
 
-                # checkin runs after the request is already torn down, therefore we add the request_data onto the
+            # checkin runs after the request is already torn down, therefore we add the request_data onto the
-                # connection_record as otherwise it won't have that information when checkin actually runs.
+            # connection_record as otherwise it won't have that information when checkin actually runs.
-                # Note: this is not a problem for checkouts as the checkout always happens within a web request or task
+            # Note: this is not a problem for checkouts as the checkout always happens within a web request or task
 
-                # web requests
+            # web requests
-                if has_request_context():
+            if has_request_context():
-                    connection_record.info['request_data'] = {
+                connection_record.info['request_data'] = {
-                        'method': request.method,
+                    'method': request.method,
-                        'host': request.host,
+                    'host': request.host,
-                        'url_rule': request.url_rule.rule if request.url_rule else 'No endpoint'
+                    'url_rule': request.url_rule.rule if request.url_rule else 'No endpoint'
-                    }
+                }
-                # celery apps
+            # celery apps
-                elif current_task:
+            elif current_task:
-                    connection_record.info['request_data'] = {
+                connection_record.info['request_data'] = {
-                        'method': 'celery',
+                    'method': 'celery',
-                        'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
+                    'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
-                        'url_rule': current_task.name,  # task name
+                    'url_rule': current_task.name,  # task name
-                    }
+                }
-                # anything else. migrations possibly, or flask cli commands.
+            # anything else. migrations possibly.
-                else:
+            else:
-                    current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
+                current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
-                    connection_record.info['request_data'] = {
+                connection_record.info['request_data'] = {
-                        'method': 'unknown',
+                    'method': 'unknown',
-                        'host': 'unknown',
+                    'host': 'unknown',
-                        'url_rule': 'unknown',
+                    'url_rule': 'unknown',
-                    }
+                }
-            except Exception:
-                current_app.logger.exception("Exception caught for checkout event.")
 
         @event.listens_for(db.engine, 'checkin')
         def checkin(dbapi_connection, connection_record):
-            try:
+            # connection returned by a web worker
-                # connection returned by a web worker
+            TOTAL_CHECKED_OUT_DB_CONNECTIONS.dec()
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.dec()
 
-                # duration that connection was held by a single web request
+            # duration that connection was held by a single web request
-                duration = time.monotonic() - connection_record.info['checkout_at']
+            duration = time.monotonic() - connection_record.info['checkout_at']
 
-                DB_CONNECTION_OPEN_DURATION_SECONDS.labels(
+            DB_CONNECTION_OPEN_DURATION_SECONDS.labels(
-                    connection_record.info['request_data']['method'],
+                connection_record.info['request_data']['method'],
-                    connection_record.info['request_data']['host'],
+                connection_record.info['request_data']['host'],
-                    connection_record.info['request_data']['url_rule']
+                connection_record.info['request_data']['url_rule']
-                ).observe(duration)
+            ).observe(duration)
-            except Exception:
-                current_app.logger.exception("Exception caught for checkin event.")

app/accept_invite/rest.py

@@ -0,0 +1,52 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    current_app
+)
+
+from itsdangerous import SignatureExpired, BadData
+
+from notifications_utils.url_safe_token import check_token
+
+from app.dao.invited_user_dao import get_invited_user_by_id
+from app.dao.organisation_dao import dao_get_invited_organisation_user
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
+from app.schemas import invited_user_schema
+
+
+accept_invite = Blueprint('accept_invite', __name__)
+register_errors(accept_invite)
+
+
+@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
+def validate_invitation_token(invitation_type, token):
+
+    max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
+
+    try:
+        invited_user_id = check_token(token,
+                                      current_app.config['SECRET_KEY'],
+                                      current_app.config['DANGEROUS_SALT'],
+                                      max_age_seconds)
+    except SignatureExpired:
+        errors = {'invitation':
+                  ['Your invitation to GOV.UK Notify has expired. '
+                   'Please ask the person that invited you to send you another one']}
+        raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
+
+    if invitation_type == 'service':
+        invited_user = get_invited_user_by_id(invited_user_id)
+        return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+    elif invitation_type == 'organisation':
+        invited_user = dao_get_invited_organisation_user(invited_user_id)
+        return jsonify(data=invited_user.serialize()), 200
+    else:
+        raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))

app/authentication/auth.py

@@ -1,22 +1,15 @@
-import uuid
+from flask import request, _request_ctx_stack, current_app, g
-
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
-from flask import current_app, g, request
-from gds_metrics import Histogram
-from notifications_python_client.authentication import (
-    decode_jwt_token,
-    get_token_issuer,
-)
 from notifications_python_client.errors import (
-    TokenAlgorithmError,
+    TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError, TokenError
-    TokenDecodeError,
-    TokenError,
-    TokenExpiredError,
-    TokenIssuerError,
 )
 from notifications_utils import request_helper
+from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
+from gds_metrics import Histogram
+
+from app.dao.services_dao import dao_fetch_service_by_id_with_api_keys
 
-from app.serialised_models import SerialisedService
 
 GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa
 
@@ -49,115 +42,7 @@ class AuthError(Exception):
         }
 
 
-class InternalApiKey():
+def get_auth_token(req):
-    def __init__(self, client_id, secret):
-        self.secret = secret
-        self.id = client_id
-        self.expiry_date = None
-
-
-def requires_no_auth():
-    pass
-
-
-def requires_govuk_alerts_auth():
-    requires_internal_auth(current_app.config.get('GOVUK_ALERTS_CLIENT_ID'))
-
-
-def requires_admin_auth():
-    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
-
-
-def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
-        raise TypeError("Unknown client_id for internal auth")
-
-    request_helper.check_proxy_header_before_request()
-    auth_token = _get_auth_token(request)
-    client_id = _get_token_issuer(auth_token)
-
-    if client_id != expected_client_id:
-        current_app.logger.info('client_id: %s', client_id)
-        current_app.logger.info('expected_client_id: %s', expected_client_id)
-        raise AuthError("Unauthorized: not allowed to perform this action", 401)
-
-    api_keys = [
-        InternalApiKey(client_id, secret)
-        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
-    ]
-
-    _decode_jwt_token(auth_token, api_keys, client_id)
-    g.service_id = client_id
-
-
-def requires_auth():
-    request_helper.check_proxy_header_before_request()
-
-    auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
-
-    try:
-        service_id = uuid.UUID(issuer)
-    except Exception:
-        raise AuthError("Invalid token: service id is not the right data type", 403)
-
-    try:
-        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
-            service = SerialisedService.from_id(service_id)
-    except NoResultFound:
-        raise AuthError("Invalid token: service not found", 403)
-
-    if not service.api_keys:
-        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
-
-    if not service.active:
-        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
-
-    api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
-
-    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
-        service_id,
-        api_key.id,
-        request.headers.get('User-Agent'),
-        request.base_url
-    ))
-
-    g.api_user = api_key
-    g.service_id = service_id
-    g.authenticated_service = service
-
-
-def _decode_jwt_token(auth_token, api_keys, service_id=None):
-    for api_key in api_keys:
-        try:
-            decode_jwt_token(auth_token, api_key.secret)
-        except TokenExpiredError:
-            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenAlgorithmError:
-            err_msg = "Invalid token: algorithm used is not HS256"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenDecodeError:
-            # we attempted to validate the token but it failed meaning it was not signed using this api key.
-            # Let's try the next one
-            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
-            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
-            # continue on to check the next API key
-            continue
-        except TokenError:
-            # General error when trying to decode and validate the token
-            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
-
-        if api_key.expiry_date:
-            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
-
-        return api_key
-    else:
-        # service has API keys, but none matching the one the user provided
-        raise AuthError("Invalid token: API key not found", 403, service_id=service_id)
-
-
-def _get_auth_token(req):
     auth_header = req.headers.get('Authorization', None)
     if not auth_header:
         raise AuthError('Unauthorized: authentication token must be provided', 401)
@@ -170,7 +55,97 @@ def _get_auth_token(req):
     return auth_header[7:]
 
 
-def _get_token_issuer(auth_token):
+def requires_no_auth():
+    pass
+
+
+def requires_admin_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
+
+    if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
+        g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
+
+        for secret in current_app.config.get('API_INTERNAL_SECRETS'):
+            try:
+                decode_jwt_token(auth_token, secret)
+                return
+            except TokenExpiredError:
+                raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
+            except TokenDecodeError:
+                # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions
+                # (which are children of `TokenDecodeError`) as these should cause an auth error immediately rather
+                # than continue on to check the next admin client secret
+                continue
+
+        # Either there are no admin client secrets or their token didn't match one of them so error
+        raise AuthError("Unauthorized: admin authentication token not found", 401)
+    else:
+        raise AuthError('Unauthorized: admin authentication token required', 401)
+
+
+def requires_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    issuer = __get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
+
+    try:
+        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
+            service = dao_fetch_service_by_id_with_api_keys(issuer)
+    except DataError:
+        raise AuthError("Invalid token: service id is not the right data type", 403)
+    except NoResultFound:
+        raise AuthError("Invalid token: service not found", 403)
+
+    if not service.api_keys:
+        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
+
+    if not service.active:
+        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
+
+    for api_key in service.api_keys:
+        try:
+            decode_jwt_token(auth_token, api_key.secret)
+        except TokenExpiredError:
+            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenAlgorithmError:
+            err_msg = "Invalid token: algorithm used is not HS256"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenDecodeError:
+            # we attempted to validate the token but it failed meaning it was not signed using this api key.
+            # Let's try the next one
+            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
+            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
+            # continue on to check the next API key
+            continue
+        except TokenError:
+            # General error when trying to decode and validate the token
+            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service.id, api_key_id=api_key.id)
+
+        if api_key.expiry_date:
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
+
+        g.service_id = api_key.service_id
+        _request_ctx_stack.top.authenticated_service = service
+        _request_ctx_stack.top.api_user = api_key
+
+        current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
+            service.id,
+            api_key.id,
+            request.headers.get('User-Agent'),
+            request.base_url
+        ))
+        return
+    else:
+        # service has API keys, but none matching the one the user provided
+        raise AuthError("Invalid token: API key not found", 403, service_id=service.id)
+
+
+def __get_token_issuer(auth_token):
     try:
         issuer = get_token_issuer(auth_token)
     except TokenIssuerError:

app/aws/s3.py

@@ -1,7 +1,8 @@
-import botocore
-from boto3 import client, resource
 from flask import current_app
 
+from boto3 import client, resource
+import botocore
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
@@ -69,6 +70,22 @@ def remove_contact_list_from_s3(service_id, contact_list_id):
     return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))
 
 
+def get_s3_bucket_objects(bucket_name, subfolder=''):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        if page.get('Contents'):
+            all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
+
+
 def remove_s3_object(bucket_name, object_key):
     obj = get_s3_object(bucket_name, object_key)
     return obj.delete()

app/billing/billing_schemas.py

@@ -1,45 +1,42 @@
 from datetime import datetime
 
+
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
     "type": "object",
     "title": "Create",
     "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
     },
     "required": ["free_sms_fragment_limit"]
 }
 
 
-def serialize_ft_billing_remove_emails(rows):
+def serialize_ft_billing_remove_emails(data):
-    return [
+    results = []
-        {
+    billed_notifications = [x for x in data if x.notification_type != 'email']
-            "month": (datetime.strftime(row.month, "%B")),
+    for notification in billed_notifications:
-            "notification_type": row.notification_type,
+        json_result = {
-            "chargeable_units": row.chargeable_units,
+            "month": (datetime.strftime(notification.month, "%B")),
-            "notifications_sent": row.notifications_sent,
+            "notification_type": notification.notification_type,
-            "rate": float(row.rate),
+            "billing_units": notification.billable_units,
-            "postage": row.postage,
+            "rate": float(notification.rate),
-            "cost": float(row.cost),
+            "postage": notification.postage,
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
         }
-        for row in rows
+        results.append(json_result)
-        if row.notification_type != 'email'
+    return results
-    ]
 
 
-def serialize_ft_billing_yearly_totals(rows):
+def serialize_ft_billing_yearly_totals(data):
-    return [
+    yearly_totals = []
-        {
+    for total in data:
-            "notification_type": row.notification_type,
+        json_result = {
-            "chargeable_units": row.chargeable_units,
+            "notification_type": total.notification_type,
-            "notifications_sent": row.notifications_sent,
+            "billing_units": total.billable_units,
-            "rate": float(row.rate),
+            "rate": float(total.rate),
-            "cost": float(row.cost),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
         }
-        for row in rows
+        yearly_totals.append(json_result)
-    ]
+
+    return yearly_totals

app/billing/rest.py

@@ -6,18 +6,18 @@ from app.billing.billing_schemas import (
     serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
     dao_get_free_sms_fragment_limit_for_year,
-    dao_update_annual_billing_for_future_years,
+    dao_get_all_free_sms_fragment_limit,
-    set_default_free_allowance_for_service,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
-    fetch_billing_totals_for_year,
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
-    fetch_monthly_billing_for_year,
 )
+
+from app.errors import InvalidRequest
 from app.errors import register_errors
-from app.models import Service
 from app.schema_validation import validate
 
 billing_blueprint = Blueprint(
@@ -30,6 +30,7 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
+@billing_blueprint.route('/ft-monthly-usage')
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
@@ -41,6 +42,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
+@billing_blueprint.route('/ft-yearly-usage-summary')
 @billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:
@@ -61,14 +63,27 @@ def get_free_sms_fragment_limit(service_id):
     annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
     if annual_billing is None:
-        service = Service.query.get(service_id)
+        # An entry does not exist in annual_billing table for that service and year. If it is a past year,
-        # An entry does not exist in annual_billing table for that service and year.
+        # we return the oldest entry.
-        # Set the annual billing to the default free allowance based on the organisation type of the service.
+        # If it is the current or future years, we create an entry in the db table using the newest record,
+        # and return that number.  If all fails, we return InvalidRequest.
+        sms_list = dao_get_all_free_sms_fragment_limit(service_id)
 
-        annual_billing = set_default_free_allowance_for_service(
+        if not sms_list:
-            service=service,
+            raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
-            year_start=int(financial_year_start) if financial_year_start else None
+        else:
-        )
+            if financial_year_start is None:
+                financial_year_start = get_current_financial_year_start_year()
+
+            if int(financial_year_start) < get_current_financial_year_start_year():
+                # return the earliest historical entry
+                annual_billing = sms_list[0]   # The oldest entry
+            else:
+                annual_billing = sms_list[-1]  # The newest entry
+
+                annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
+                                                                              annual_billing.free_sms_fragment_limit,
+                                                                              financial_year_start)
 
     return jsonify(annual_billing.serialize_free_sms_items()), 200
 

app/broadcast_message/broadcast_message_schema.py

@@ -1,60 +0,0 @@
-from app.models import BroadcastStatusType
-from app.schema_validation.definitions import uuid
-
-create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST create broadcast_message schema',
-    'type': 'object',
-    'title': 'Create broadcast_message',
-    'properties': {
-        'template_id': uuid,
-        'service_id': uuid,
-        'created_by': uuid,
-        'personalisation': {'type': 'object'},
-        'starts_at': {'type': 'string', 'format': 'datetime'},
-        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-        'content': {'type': 'string', 'minLength': 1},
-        'reference': {'type': 'string', 'minLength': 1, 'maxLength': 255},
-    },
-    'required': ['service_id', 'created_by'],
-    'allOf': [
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['content']},
-        ]},
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['reference']},
-        ]},
-    ],
-    'additionalProperties': False
-}
-
-update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST update broadcast_message schema',
-    'type': 'object',
-    'title': 'Update broadcast_message',
-    'properties': {
-        'personalisation': {'type': 'object'},
-        'starts_at': {'type': 'string', 'format': 'datetime'},
-        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-    },
-    'required': [],
-    'additionalProperties': False
-}
-
-update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST update broadcast_message status schema',
-    'type': 'object',
-    'title': 'Update broadcast_message',
-    'properties': {
-        'status': {'type': 'string', 'enum': BroadcastStatusType.STATUSES},
-        'created_by': uuid,
-    },
-    'required': ['status', 'created_by'],
-    'additionalProperties': False
-}

app/broadcast_message/rest.py

@@ -1,165 +0,0 @@
-import iso8601
-from flask import Blueprint, jsonify, request
-from notifications_utils.template import BroadcastMessageTemplate
-
-from app.broadcast_message import utils as broadcast_utils
-from app.broadcast_message.broadcast_message_schema import (
-    create_broadcast_message_schema,
-    update_broadcast_message_schema,
-    update_broadcast_message_status_schema,
-)
-from app.dao.broadcast_message_dao import (
-    dao_get_broadcast_message_by_id_and_service_id,
-    dao_get_broadcast_messages_for_service,
-)
-from app.dao.dao_utils import dao_save_object
-from app.dao.services_dao import dao_fetch_service_by_id
-from app.dao.templates_dao import dao_get_template_by_id_and_service_id
-from app.dao.users_dao import get_user_by_id
-from app.errors import InvalidRequest, register_errors
-from app.models import BroadcastMessage, BroadcastStatusType
-from app.schema_validation import validate
-
-broadcast_message_blueprint = Blueprint(
-    'broadcast_message',
-    __name__,
-    url_prefix='/service/<uuid:service_id>/broadcast-message'
-)
-register_errors(broadcast_message_blueprint)
-
-
-def _parse_nullable_datetime(dt):
-    if dt:
-        return iso8601.parse_date(dt).replace(tzinfo=None)
-    return dt
-
-
-@broadcast_message_blueprint.route('', methods=['GET'])
-def get_broadcast_messages_for_service(service_id):
-    # TODO: should this return template content/data in some way? or can we rely on them being cached admin side.
-    # we might need stuff like template name for showing on the dashboard.
-    # TODO: should this paginate or filter on dates or anything?
-    broadcast_messages = [o.serialize() for o in dao_get_broadcast_messages_for_service(service_id)]
-    return jsonify(broadcast_messages=broadcast_messages)
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['GET'])
-def get_broadcast_message(service_id, broadcast_message_id):
-    return jsonify(dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id).serialize())
-
-
-@broadcast_message_blueprint.route('', methods=['POST'])
-def create_broadcast_message(service_id):
-    data = request.get_json()
-
-    validate(data, create_broadcast_message_schema)
-    service = dao_fetch_service_by_id(data['service_id'])
-    user = get_user_by_id(data['created_by'])
-    personalisation = data.get('personalisation', {})
-    template_id = data.get('template_id')
-
-    if template_id:
-        template = dao_get_template_by_id_and_service_id(
-            template_id, data['service_id']
-        )
-        content = str(template._as_utils_template_with_personalisation(
-            personalisation
-        ))
-        reference = None
-    else:
-        temporary_template = BroadcastMessageTemplate.from_content(data['content'])
-        if temporary_template.content_too_long:
-            raise InvalidRequest(
-                (
-                    f'Content must be '
-                    f'{temporary_template.max_content_count:,.0f} '
-                    f'characters or fewer'
-                ) + (
-                    ' (because it could not be GSM7 encoded)'
-                    if temporary_template.non_gsm_characters else ''
-                ),
-                status_code=400,
-            )
-        template = None
-        content = str(temporary_template)
-        reference = data['reference']
-
-    broadcast_message = BroadcastMessage(
-        service_id=service.id,
-        template_id=template_id,
-        template_version=template.version if template else None,
-        personalisation=personalisation,
-        areas=data.get("areas", {}),
-        status=BroadcastStatusType.DRAFT,
-        starts_at=_parse_nullable_datetime(data.get('starts_at')),
-        finishes_at=_parse_nullable_datetime(data.get('finishes_at')),
-        created_by_id=user.id,
-        content=content,
-        reference=reference,
-        stubbed=service.restricted
-    )
-
-    dao_save_object(broadcast_message)
-
-    return jsonify(broadcast_message.serialize()), 201
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['POST'])
-def update_broadcast_message(service_id, broadcast_message_id):
-    data = request.get_json()
-    validate(data, update_broadcast_message_schema)
-
-    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
-
-    if broadcast_message.status not in BroadcastStatusType.PRE_BROADCAST_STATUSES:
-        raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id} while it has status {broadcast_message.status}',
-            status_code=400
-        )
-
-    areas = data.get("areas", {})
-
-    if ('ids' in areas and 'simple_polygons' not in areas) or ('ids' not in areas and 'simple_polygons' in areas):
-        raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.',
-            status_code=400
-        )
-
-    if 'personalisation' in data:
-        broadcast_message.personalisation = data['personalisation']
-    if 'starts_at' in data:
-        broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at'])
-    if 'finishes_at' in data:
-        broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at'])
-    if 'ids' in areas and 'simple_polygons' in areas:
-        broadcast_message.areas = areas
-
-    dao_save_object(broadcast_message)
-
-    return jsonify(broadcast_message.serialize()), 200
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>/status', methods=['POST'])
-def update_broadcast_message_status(service_id, broadcast_message_id):
-    data = request.get_json()
-
-    validate(data, update_broadcast_message_status_schema)
-    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
-
-    if not broadcast_message.service.active:
-        raise InvalidRequest("Updating broadcast message is not allowed: service is inactive ", 403)
-
-    new_status = data['status']
-    updating_user = get_user_by_id(data['created_by'])
-
-    if updating_user not in broadcast_message.service.users:
-        #  we allow platform admins to cancel broadcasts, and we don't check user if request was done via API
-        if not (new_status == BroadcastStatusType.CANCELLED and updating_user.platform_admin):
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot update broadcast_message {broadcast_message.id} from other service',
-                status_code=400
-            )
-
-    broadcast_utils.update_broadcast_message_status(broadcast_message, new_status, updating_user)
-
-    return jsonify(broadcast_message.serialize()), 200

app/broadcast_message/translators.py

@@ -1,37 +0,0 @@
-from bs4 import BeautifulSoup
-
-
-def cap_xml_to_dict(cap_xml):
-    # This function assumes that it’s being passed valid CAP XML
-    cap = BeautifulSoup(cap_xml, "xml")
-    return {
-        "msgType": cap.alert.msgType.text,
-        "reference": cap.alert.identifier.text,
-        "references": (
-            # references to previous events belonging to the same alert
-            cap.alert.references.text if cap.alert.references else None
-        ),
-        "cap_event": cap.alert.info.event.text,
-        "category": cap.alert.info.category.text,
-        "expires": cap.alert.info.expires.text,
-        "content": cap.alert.info.description.text,
-        "areas": [
-            {
-                "name": area.areaDesc.text,
-                "polygons": [
-                    cap_xml_polygon_to_list(polygon.text)
-                    for polygon in area.find_all('polygon')
-                ]
-            }
-            for area in cap.alert.info.find_all('area')
-        ]
-    }
-
-
-def cap_xml_polygon_to_list(polygon_string):
-    return [
-        [
-            float(coordinate) for coordinate in pair.split(',')
-        ]
-        for pair in polygon_string.strip().split(' ')
-    ]

app/broadcast_message/utils.py

@@ -1,140 +0,0 @@
-import inspect
-from datetime import datetime
-
-from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-
-from app import zendesk_client
-from app.celery.broadcast_message_tasks import send_broadcast_event
-from app.config import QueueNames
-from app.dao.dao_utils import dao_save_object
-from app.errors import InvalidRequest
-from app.models import (
-    BroadcastEvent,
-    BroadcastEventMessageType,
-    BroadcastStatusType,
-)
-
-
-def update_broadcast_message_status(broadcast_message, new_status, updating_user=None, api_key_id=None):
-    _validate_broadcast_update(broadcast_message, new_status, updating_user)
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        broadcast_message.approved_at = datetime.utcnow()
-        broadcast_message.approved_by = updating_user
-
-    if new_status == BroadcastStatusType.CANCELLED:
-        broadcast_message.cancelled_at = datetime.utcnow()
-        broadcast_message.cancelled_by = updating_user
-        broadcast_message.cancelled_by_api_key_id = api_key_id
-
-    current_app.logger.info(
-        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
-    )
-    broadcast_message.status = new_status
-
-    dao_save_object(broadcast_message)
-    _create_p1_zendesk_alert(broadcast_message)
-
-    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
-        _create_broadcast_event(broadcast_message)
-
-
-def _validate_broadcast_update(broadcast_message, new_status, updating_user):
-    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
-        raise InvalidRequest(
-            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
-            status_code=400
-        )
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        # training mode services can approve their own broadcasts
-        if updating_user == broadcast_message.created_by and not broadcast_message.service.restricted:
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
-                status_code=400
-            )
-        elif len(broadcast_message.areas['simple_polygons']) == 0:
-            raise InvalidRequest(
-                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
-                status_code=400
-            )
-
-
-def _create_p1_zendesk_alert(broadcast_message):
-    if current_app.config['NOTIFY_ENVIRONMENT'] != 'live':
-        return
-
-    if broadcast_message.status != BroadcastStatusType.BROADCASTING:
-        return
-
-    message = inspect.cleandoc(f"""
-        Broadcast Sent
-
-        https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}
-
-        Sent on channel {broadcast_message.service.broadcast_channel} to {broadcast_message.areas["names"]}.
-
-        Content starts "{broadcast_message.content[:100]}".
-
-        Follow the runbook to check the broadcast went out OK:
-        https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs/edit#heading=h.lzr9aq5b4wg
-    """)
-
-    ticket = NotifySupportTicket(
-        subject='Live broadcast sent',
-        message=message,
-        ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-        technical_ticket=True,
-        org_id=current_app.config['BROADCAST_ORGANISATION_ID'],
-        org_type='central',
-        service_id=str(broadcast_message.service_id),
-        p1=True
-    )
-    zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-def _create_broadcast_event(broadcast_message):
-    """
-    If the service is live and the broadcast message is not stubbed, creates a broadcast event, stores it in the
-    database, and triggers the task to send the CAP XML off.
-    """
-    service = broadcast_message.service
-
-    if not broadcast_message.stubbed and not service.restricted:
-        msg_types = {
-            BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
-            BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
-        }
-
-        event = BroadcastEvent(
-            service=service,
-            broadcast_message=broadcast_message,
-            message_type=msg_types[broadcast_message.status],
-            transmitted_content={"body": broadcast_message.content},
-            transmitted_areas=broadcast_message.areas,
-            # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it
-            # include a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
-            transmitted_sender='notifications.service.gov.uk',
-
-            # TODO: Should this be set to now? Or the original starts_at?
-            transmitted_starts_at=broadcast_message.starts_at,
-            transmitted_finishes_at=broadcast_message.finishes_at,
-        )
-
-        dao_save_object(event)
-
-        send_broadcast_event.apply_async(
-            kwargs={'broadcast_event_id': str(event.id)},
-            queue=QueueNames.BROADCASTS
-        )
-    elif broadcast_message.stubbed != service.restricted:
-        # It's possible for a service to create a broadcast in trial mode, and then approve it after the
-        # service is live (or vice versa). We don't think it's safe to send such broadcasts, as the service
-        # has changed since they were created. Log an error instead.
-        current_app.logger.error(
-            f'Broadcast event not created. Stubbed status of broadcast message was {broadcast_message.stubbed}'
-            f' but service was {"in trial mode" if service.restricted else "live"}'
-        )

app/celery/broadcast_message_tasks.py

@@ -1,230 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import cbc_proxy_client, notify_celery
-from app.clients.cbc_proxy import CBCProxyRetryableException
-from app.config import QueueNames, TaskNames
-from app.dao.broadcast_message_dao import (
-    create_broadcast_provider_message,
-    dao_get_broadcast_event_by_id,
-    update_broadcast_provider_message_status,
-)
-from app.models import (
-    BroadcastEventMessageType,
-    BroadcastProvider,
-    BroadcastProviderMessageStatus,
-)
-from app.utils import format_sequential_number
-
-
-class BroadcastIntegrityError(Exception):
-    pass
-
-
-def get_retry_delay(retry_count):
-    """
-    Given a count of retries so far, return a delay for the next one.
-    `retry_count` should be 0 the first time a task fails.
-    """
-    # TODO: replace with celery's built in exponential backoff
-
-    # 2 to the power of x. 1, 2, 4, 8, 16, 32, ...
-    delay = 2**retry_count
-    # never wait longer than 4 minutes
-    return min(delay, 240)
-
-
-def check_event_is_authorised_to_be_sent(broadcast_event, provider):
-    if not broadcast_event.service.active:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is suspended'
-        )
-
-    if broadcast_event.service.restricted:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is not live'
-        )
-
-    if broadcast_event.broadcast_message.stubbed:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the broadcast message is stubbed'
-        )
-
-
-def check_event_makes_sense_in_sequence(broadcast_event, provider):
-    """
-    If any previous event hasn't sent yet for that provider, then we shouldn't send the current event. Instead, fail and
-    raise a zendesk ticket - so that a notify team member can assess the state of the previous messages, and if
-    necessary, can replay the `send_broadcast_provider_message` task if the previous message has now been sent.
-
-    Note: This is called before the new broadcast_provider_message is created.
-
-    # Help, I've come across this code following a pagerduty alert, what should I do?
-
-    1. Find the failing broadcast_provider_message associated with the previous event that caused this to trip.
-    2. If that provider message is still failing to send, fix the issue causing that. The task to send that previous
-       message might still be retrying in the background - look for logs related to that task.
-    3. If that provider message has sent succesfully, you might need to send this task off depending on context. This
-       might not always be true though, for example, it may not be necessary to send a cancel if the original alert has
-       already expired.
-    4. If you need to re-send this task off again, you'll need to run the following command on paas:
-       `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)`
-    """
-    current_provider_message = broadcast_event.get_provider_message(provider)
-    # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'It is in status {current_provider_message.status}'
-        )
-
-    if broadcast_event.transmitted_finishes_at < datetime.utcnow():
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'The expiry time of {broadcast_event.transmitted_finishes_at} has already passed'
-        )
-
-    # get events sorted from earliest to latest
-    events = sorted(broadcast_event.broadcast_message.events, key=lambda x: x.sent_at)
-
-    for prev_event in events:
-        if prev_event.id != broadcast_event.id and prev_event.sent_at < broadcast_event.sent_at:
-            # get the record from when that event was sent to the same provider
-            prev_provider_message = prev_event.get_provider_message(provider)
-
-            # the previous message hasn't even got round to running `send_broadcast_provider_message` yet.
-            if not prev_provider_message:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has no provider_message for provider {provider} yet.\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-            # if there's a previous message that has started but not finished sending (whether it fatally errored or is
-            # currently retrying)
-            if prev_provider_message.status != BroadcastProviderMessageStatus.ACK:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has not finished sending to provider {provider} yet.\n' +
-                    f'It is currently in status "{prev_provider_message.status}".\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-
-@notify_celery.task(name="send-broadcast-event")
-def send_broadcast_event(broadcast_event_id):
-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
-
-    for provider in broadcast_event.service.get_available_broadcast_providers():
-        send_broadcast_provider_message.apply_async(
-            kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider},
-            queue=QueueNames.BROADCASTS
-        )
-
-
-# max_retries=None: retry forever
-@notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None)
-def send_broadcast_provider_message(self, broadcast_event_id, provider):
-    if not current_app.config['CBC_PROXY_ENABLED']:
-        current_app.logger.info(
-            "CBC Proxy disabled, not sending broadcast_provider_message for "
-            f"broadcast_event_id {broadcast_event_id} with provider {provider}"
-        )
-        return
-
-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    check_event_is_authorised_to_be_sent(broadcast_event, provider)
-    check_event_makes_sense_in_sequence(broadcast_event, provider)
-
-    # the broadcast_provider_message may already exist if we retried previously
-    broadcast_provider_message = broadcast_event.get_provider_message(provider)
-    if broadcast_provider_message is None:
-        broadcast_provider_message = create_broadcast_provider_message(broadcast_event, provider)
-
-    formatted_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        formatted_message_number = format_sequential_number(broadcast_provider_message.message_number)
-
-    current_app.logger.info(
-        f'Invoking cbc proxy to send broadcast_provider_message with ID of {broadcast_provider_message.id} '
-        f'and broadcast_event ID of {broadcast_event_id} '
-        f'msgType {broadcast_event.message_type}'
-    )
-
-    areas = [
-        {"polygon": polygon}
-        for polygon in broadcast_event.transmitted_areas["simple_polygons"]
-    ]
-
-    cbc_proxy_provider_client = cbc_proxy_client.get_proxy(provider)
-
-    try:
-        if broadcast_event.message_type == BroadcastEventMessageType.ALERT:
-            cbc_proxy_provider_client.create_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.UPDATE:
-            cbc_proxy_provider_client.update_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                # We think an alert update should always go out on the same channel that created the alert
-                # We recognise there is a small risk with this code here that if the services channel was
-                # changed between an alert being sent out and then updated, then something might go wrong
-                # but we are relying on service channels changing almost never, and not mid incident
-                # We may consider in the future, changing this such that we store the channel a broadcast was
-                # sent on on the broadcast message itself and pick the value from there instead of the service
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.CANCEL:
-            cbc_proxy_provider_client.cancel_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-            )
-    except CBCProxyRetryableException as exc:
-        delay = get_retry_delay(self.request.retries)
-        current_app.logger.exception(
-            f'Retrying send_broadcast_provider_message for broadcast event {broadcast_event_id}, '
-            f'provider message {broadcast_provider_message.id}, provider {provider} in {delay} seconds'
-        )
-
-        self.retry(
-            exc=exc,
-            countdown=delay,
-            queue=QueueNames.BROADCASTS,
-        )
-
-    update_broadcast_provider_message_status(broadcast_provider_message, status=BroadcastProviderMessageStatus.ACK)
-
-
-@notify_celery.task(name='trigger-link-test')
-def trigger_link_test(provider):
-    cbc_proxy_client.get_proxy(provider).send_link_test()

app/celery/celery.py

@@ -0,0 +1,78 @@
+import time
+
+from gds_metrics.metrics import Histogram
+from celery import Celery, Task
+from celery.signals import worker_process_shutdown
+from flask import g, request
+from flask.ctx import has_request_context
+
+
+@worker_process_shutdown.connect
+def log_on_worker_shutdown(sender, signal, pid, exitcode, **kwargs):
+    # imported here to avoid circular imports
+    from app import notify_celery
+
+    # if the worker has already restarted at least once, then we no longer have app context and current_app won't work
+    # to create a new one. Instead we have to create a new app context from the original flask app and use that instead.
+    with notify_celery._app.app_context():
+        # if the worker has restarted
+        notify_celery._app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
+
+
+def make_task(app):
+    SQS_APPLY_ASYNC_DURATION_SECONDS = Histogram(
+        'sqs_apply_async_duration_seconds',
+        'Time taken to put task on queue',
+        ['task_name']
+    )
+
+    class NotifyTask(Task):
+        abstract = True
+        start = None
+
+        def on_success(self, retval, task_id, args, kwargs):
+            elapsed_time = time.time() - self.start
+            app.logger.info(
+                "{task_name} took {time}".format(
+                    task_name=self.name, time="{0:.4f}".format(elapsed_time)
+                )
+            )
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            # ensure task will log exceptions to correct handlers
+            app.logger.exception('Celery task: {} failed'.format(self.name))
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def __call__(self, *args, **kwargs):
+            # ensure task has flask context to access config, logger, etc
+            with app.app_context():
+                self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
+                return super().__call__(*args, **kwargs)
+
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            with SQS_APPLY_ASYNC_DURATION_SECONDS.labels(self.name).time():
+                return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
+    return NotifyTask
+
+
+class NotifyCelery(Celery):
+
+    def init_app(self, app):
+        super().__init__(
+            app.import_name,
+            broker=app.config['BROKER_URL'],
+            task_cls=make_task(app),
+        )
+
+        self.conf.update(app.config)
+        self._app = app

app/celery/letters_pdf_tasks.py

@@ -1,47 +1,40 @@
-from base64 import urlsafe_b64encode
 from datetime import datetime, timedelta
 from hashlib import sha512
+from base64 import urlsafe_b64encode
 
 from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
+
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
-from notifications_utils.postal_address import PostalAddress
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import encryption, notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
-from app.cronitor import cronitor
 from app.dao.notifications_dao import (
-    dao_get_letters_and_sheets_volume_by_postage,
-    dao_get_letters_to_be_printed,
-    dao_get_notification_by_reference,
-    dao_update_notification,
-    dao_update_notifications_by_reference,
     get_notification_by_id,
     update_notification_status_by_id,
+    dao_update_notification,
+    dao_get_notification_by_reference,
+    dao_update_notifications_by_reference,
+    dao_get_letters_to_be_printed,
 )
-from app.dao.templates_dao import dao_get_template_by_id
+from app.letters.utils import get_letter_pdf_filename
 from app.errors import VirusScanError
 from app.exceptions import NotificationTechnicalFailureException
 from app.letters.utils import (
-    LetterPDFNotFound,
-    ScanErrorType,
-    find_letter_pdf_in_s3,
-    generate_letter_pdf_filename,
     get_billable_units_for_letter_page_count,
-    get_file_names_from_error_bucket,
-    get_folder_name,
     get_reference_from_filename,
-    move_error_pdf_to_scan_bucket,
+    ScanErrorType,
     move_failed_pdf,
     move_sanitised_letter_to_test_or_live_pdf_bucket,
     move_scan_to_invalid_pdf_bucket,
+    move_error_pdf_to_scan_bucket,
+    get_file_names_from_error_bucket,
 )
 from app.models import (
     INTERNATIONAL_LETTERS,
-    INTERNATIONAL_POSTAGE_TYPES,
-    KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
@@ -49,20 +42,28 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
-    POSTAGE_TYPES,
+    LETTER_TYPE
-    RESOLVE_POSTAGE_FOR_FILE_NAME,
-    Service,
 )
+from app.cronitor import cronitor
+
+
+@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def create_letters_pdf(self, notification_id):
+    get_pdf_for_templated_letter(notification_id)
 
 
 @notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def get_pdf_for_templated_letter(self, notification_id):
     try:
         notification = get_notification_by_id(notification_id, _raise=True)
-        letter_filename = generate_letter_pdf_filename(
+
+        letter_filename = get_letter_pdf_filename(
             reference=notification.reference,
-            created_at=notification.created_at,
+            crown=notification.service.crown,
-            ignore_folder=notification.key_type == KEY_TYPE_TEST,
+            sending_date=notification.created_at,
+            dont_use_sending_date=notification.key_type == KEY_TYPE_TEST,
             postage=notification.postage
         )
         letter_data = {
@@ -86,12 +87,12 @@ def get_pdf_for_templated_letter(self, notification_id):
             args=(encrypted_data,),
             queue=QueueNames.SANITISE_LETTERS
         )
-    except Exception as e:
+    except Exception:
         try:
             current_app.logger.exception(
                 f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
             )
-            self.retry(exc=e, queue=QueueNames.RETRY)
+            self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
             message = f"RETRY FAILED: Max retries reached. " \
                       f"The task create-letter-pdf failed for notification id {notification_id}. " \
@@ -101,6 +102,7 @@ def get_pdf_for_templated_letter(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def update_billable_units_for_letter(self, notification_id, page_count):
     notification = get_notification_by_id(notification_id, _raise=True)
 
@@ -116,16 +118,6 @@ def update_billable_units_for_letter(self, notification_id, page_count):
         )
 
 
-@notify_celery.task(
-    bind=True, name="update-validation-failed-for-templated-letter", max_retries=15, default_retry_delay=300
-)
-def update_validation_failed_for_templated_letter(self, notification_id, page_count):
-    notification = get_notification_by_id(notification_id, _raise=True)
-    notification.status = NOTIFICATION_VALIDATION_FAILED
-    dao_update_notification(notification)
-    current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}")
-
-
 @notify_celery.task(name='collate-letter-pdfs-to-be-sent')
 @cronitor("collate-letter-pdfs-to-be-sent")
 def collate_letter_pdfs_to_be_sent():
@@ -143,118 +135,58 @@ def collate_letter_pdfs_to_be_sent():
     print_run_deadline = print_run_date.replace(
         hour=17, minute=30, second=0, microsecond=0
     )
-    _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)
 
-    for postage in POSTAGE_TYPES:
+    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
-        current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
-        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
 
-        for i, letters in enumerate(group_letters(letters_to_print)):
+    for i, letters in enumerate(group_letters(letters_to_print)):
-            filenames = [letter['Key'] for letter in letters]
+        filenames = [letter['Key'] for letter in letters]
 
-            service_id = letters[0]['ServiceId']
+        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-            organisation_id = letters[0]['OrganisationId']
+        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-
+        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
-            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+            date=print_run_deadline.strftime("%Y-%m-%d"),
-            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+            num=i + 1,
-            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format(
+            hash=hash
-                date=print_run_deadline.strftime("%Y-%m-%d"),
-                postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
-                num=i + 1,
-                hash=hash,
-                service_id=service_id,
-                organisation_id=organisation_id
-            )
-
-            current_app.logger.info(
-                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                    len(filenames),
-                    dvla_filename,
-                    sum(letter['Size'] for letter in letters)
-                )
-            )
-            notify_celery.send_task(
-                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-                kwargs={
-                    'filenames_to_zip': filenames,
-                    'upload_filename': dvla_filename
-                },
-                queue=QueueNames.PROCESS_FTP,
-                compression='zlib'
-            )
-        current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}")
-
-    current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
-
-
-def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
-    letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
-    send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date())
-
-
-def send_letters_volume_email_to_dvla(letters_volumes, date):
-    personalisation = {
-        'total_volume': 0,
-        'first_class_volume': 0,
-        'second_class_volume': 0,
-        'international_volume': 0,
-        'total_sheets': 0,
-        'first_class_sheets': 0,
-        "second_class_sheets": 0,
-        'international_sheets': 0,
-        'date': date.strftime("%d %B %Y")
-    }
-    for item in letters_volumes:
-        personalisation['total_volume'] += item.letters_count
-        personalisation['total_sheets'] += item.sheets_count
-        if f"{item.postage}_class_volume" in personalisation:
-            personalisation[f"{item.postage}_class_volume"] = item.letters_count
-            personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
-        else:
-            personalisation["international_volume"] += item.letters_count
-            personalisation["international_sheets"] += item.sheets_count
-
-    template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
-    recipients = current_app.config['DVLA_EMAIL_ADDRESSES']
-    reply_to = template.service.get_default_reply_to_email_address()
-    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
-
-    # avoid circular imports:
-    from app.notifications.process_notifications import (
-        persist_notification,
-        send_notification_to_queue,
-    )
-    for recipient in recipients:
-        saved_notification = persist_notification(
-            template_id=template.id,
-            template_version=template.version,
-            recipient=recipient,
-            service=service,
-            personalisation=personalisation,
-            notification_type=template.template_type,
-            api_key_id=None,
-            key_type=KEY_TYPE_NORMAL,
-            reply_to_text=reply_to
         )
 
-        send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+        current_app.logger.info(
+            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                len(filenames),
+                dvla_filename,
+                sum(letter['Size'] for letter in letters)
+            )
+        )
+        notify_celery.send_task(
+            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+            kwargs={
+                'filenames_to_zip': filenames,
+                'upload_filename': dvla_filename
+            },
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
+        )
 
 
-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
+
+    letter_pdfs = []
     for letter in letters_awaiting_sending:
         try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
+            letter_file_name = get_letter_pdf_filename(
-            yield {
+                reference=letter.reference,
-                "Key": letter_pdf.key,
+                crown=letter.service.crown,
-                "Size": letter_pdf.size,
+                sending_date=letter.created_at,
-                "ServiceId": str(letter.service_id),
+                postage=letter.postage
-                "OrganisationId": str(letter.service.organisation_id)
+            )
-            }
+            letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
-        except (BotoClientError, LetterPDFNotFound) as e:
+            letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
+        except BotoClientError as e:
             current_app.logger.exception(
                 f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
 
+    return letter_pdfs
+
 
 def group_letters(letter_pdfs):
     """
@@ -264,23 +196,16 @@ def group_letters(letter_pdfs):
     """
     running_filesize = 0
     list_of_files = []
-    service_id = None
     for letter in letter_pdfs:
         if letter['Key'].lower().endswith('.pdf'):
-            if not service_id:
-                service_id = letter['ServiceId']
             if (
-                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
-                or len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
-                or letter['ServiceId'] != service_id
             ):
                 yield list_of_files
                 running_filesize = 0
                 list_of_files = []
-                service_id = None
 
-            if not service_id:
-                service_id = letter['ServiceId']
             running_filesize += letter['Size']
             list_of_files.append(letter)
 
@@ -292,7 +217,7 @@ def group_letters(letter_pdfs):
 def sanitise_letter(self, filename):
     try:
         reference = get_reference_from_filename(filename)
-        notification = dao_get_notification_by_reference(reference)
+        notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
 
         current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))
 
@@ -374,22 +299,7 @@ def process_sanitised_letter(self, sanitise_data):
             billable_units=billable_units,
             recipient_address=letter_details['address']
         )
-
+        move_sanitised_letter_to_test_or_live_pdf_bucket(filename, is_test_key, notification.created_at)
-        # The original filename could be wrong because we didn't know the postage.
-        # Now we know if the letter is international, we can check what the filename should be.
-        upload_file_name = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-        move_sanitised_letter_to_test_or_live_pdf_bucket(
-            filename,
-            is_test_key,
-            notification.created_at,
-            upload_file_name,
-        )
         # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
         original_pdf_object.delete()
 
@@ -443,7 +353,7 @@ def _move_invalid_letter_and_update_status(
 def process_virus_scan_failed(filename):
     move_failed_pdf(filename, ScanErrorType.FAILURE)
     reference = get_reference_from_filename(filename)
-    notification = dao_get_notification_by_reference(reference)
+    notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0)
 
     if updated_count != 1:
@@ -462,7 +372,7 @@ def process_virus_scan_failed(filename):
 def process_virus_scan_error(filename):
     move_failed_pdf(filename, ScanErrorType.ERROR)
     reference = get_reference_from_filename(filename)
-    notification = dao_get_notification_by_reference(reference)
+    notification = dao_get_notification_by_reference(reference=reference, notification_type=LETTER_TYPE)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE, billable_units=0)
 
     if updated_count != 1:
@@ -477,19 +387,10 @@ def process_virus_scan_error(filename):
 
 
 def update_letter_pdf_status(reference, status, billable_units, recipient_address=None):
-    postage = None
+
-    if recipient_address:
-        # fix allow_international_letters
-        postage = PostalAddress(raw_address=recipient_address.replace(',', '\n'),
-                                allow_international_letters=True
-                                ).postage
-        postage = postage if postage in INTERNATIONAL_POSTAGE_TYPES else None
     update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()}
-    if postage:
-        update_dict.update({'postage': postage, 'international': True})
     if recipient_address:
         update_dict['to'] = recipient_address
-        update_dict['normalised_to'] = ''.join(recipient_address.split()).lower()
     return dao_update_notifications_by_reference(
         references=[reference],
         update_dict=update_dict)[0]
@@ -534,37 +435,3 @@ def replay_letters_in_error(filename=None):
                     [filename],
                     queue=QueueNames.LETTERS
                 )
-
-
-@notify_celery.task(name='resanitise-pdf')
-def resanitise_pdf(notification_id):
-    """
-    `notification_id` is the notification id for a PDF letter which was either uploaded or sent using the API.
-
-    This task calls the `recreate_pdf_for_precompiled_letter` template preview task which recreates the
-    PDF for a letter which is already sanitised and in the letters-pdf bucket. The new file that is generated
-    will then overwrite the existing letter in the letters-pdf bucket.
-    """
-    notification = get_notification_by_id(notification_id)
-
-    # folder_name is the folder that the letter is in the letters-pdf bucket e.g. '2021-10-10/'
-    folder_name = get_folder_name(notification.created_at)
-
-    filename = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-    notify_celery.send_task(
-        name=TaskNames.RECREATE_PDF_FOR_PRECOMPILED_LETTER,
-        kwargs={
-            'notification_id': str(notification.id),
-            'file_location': f'{folder_name}{filename}',
-            'allow_international_letters': notification.service.has_permission(
-                INTERNATIONAL_LETTERS
-            ),
-        },
-        queue=QueueNames.SANITISE_LETTERS,
-    )

app/celery/nightly_tasks.py

@@ -1,56 +1,55 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
 import pytz
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
+from notifications_utils.statsd_decorators import statsd
-    NotifySupportTicket,
-)
-from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, statsd_client, zendesk_client
+from app import notify_celery, performance_platform_client, zendesk_client
 from app.aws import s3
+from app.celery.service_callback_tasks import (
+    send_delivery_status_to_service,
+    create_delivery_status_callback_data,
+)
 from app.config import QueueNames
-from app.cronitor import cronitor
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
-    dao_archive_job,
     dao_get_jobs_older_than_data_retention,
+    dao_archive_job
 )
 from app.dao.notifications_dao import (
-    dao_get_notifications_processing_time_stats,
     dao_timeout_notifications,
-    get_service_ids_with_notifications_before,
+    delete_notifications_older_than_retention_by_type,
-    move_notifications_to_notification_history,
-)
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_for_all_services_by_notification_type,
 )
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_SENDING,
-    SMS_TYPE,
-    FactProcessingTime,
     Notification,
+    NOTIFICATION_SENDING,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    KEY_TYPE_NORMAL
 )
-from app.notifications.notifications_ses_callback import (
+from app.performance_platform import total_sent_notifications, processing_time
-    check_and_queue_callback_task,
+from app.cronitor import cronitor
-)
 from app.utils import get_london_midnight_in_utc
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
+@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
     _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
 
 
 @notify_celery.task(name="remove_letter_jobs")
 @cronitor("remove_letter_jobs")
+@statsd(namespace="tasks")
 def remove_letter_csv_files():
     _remove_csv_files([LETTER_TYPE])
 
@@ -64,111 +63,149 @@ def _remove_csv_files(job_types):
 
 
 @notify_celery.task(name="delete-notifications-older-than-retention")
+@statsd(namespace="tasks")
 def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_email_notifications_older_than_retention()
-    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_sms_notifications_older_than_retention()
-    delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_letter_notifications_older_than_retention()
 
 
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
+@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('sms')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('sms')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise
 
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
+@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('email')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('email')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete email notifications")
+        raise
 
 
 @notify_celery.task(name="delete-letter-notifications")
 @cronitor("delete-letter-notifications")
+@statsd(namespace="tasks")
 def delete_letter_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('letter')
+    try:
-
+        start = datetime.utcnow()
-
+        deleted = delete_notifications_older_than_retention_by_type('letter')
-def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
-
-    for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_london_midnight_in_utc(
-            convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
-        )
-
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': f.service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': day_to_delete_backwards_from
-        })
-
-    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7))
-    service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
-
-    # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
-    # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
-        notification_type,
-        seven_days_ago
-    )
-
-    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
-
-    for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': seven_days_ago
-        })
-
-    current_app.logger.info(
-        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
-        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
-        f'{len(service_ids_to_purge)} services without flexible data retention'
-    )
-
-
-@notify_celery.task(name='delete-notifications-for-service-and-type')
-def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
-    start = datetime.utcnow()
-    num_deleted = move_notifications_to_notification_history(
-        notification_type,
-        service_id,
-        datetime_to_delete_before,
-    )
-    if num_deleted:
-        end = datetime.utcnow()
         current_app.logger.info(
-            f'delete-notifications-for-service-and-type: '
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
-            f'service: {service_id}, '
+                'letter',
-            f'notification_type: {notification_type}, '
+                start,
-            f'count deleted: {num_deleted}, '
+                datetime.utcnow(),
-            f'duration: {(end - start).seconds} seconds'
+                deleted
+            )
         )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete letter notifications")
+        raise
 
 
 @notify_celery.task(name='timeout-sending-notifications')
 @cronitor('timeout-sending-notifications')
+@statsd(namespace="tasks")
 def timeout_notifications():
-    notifications = ['dummy value so len() > 0']
+    technical_failure_notifications, temporary_failure_notifications = \
+        dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
 
-    cutoff_time = datetime.utcnow() - timedelta(
+    notifications = technical_failure_notifications + temporary_failure_notifications
-        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
+    for notification in notifications:
+        # queue callback task only if the service_callback_api exists
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
+
+    current_app.logger.info(
+        "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    if technical_failure_notifications:
+        message = "{} notifications have been updated to technical-failure because they " \
+                  "have timed out and are still in created.Notification ids: {}".format(
+                      len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
+        raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@cronitor('send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_platform_stats(date=None):
+    # date is a string in the format of "YYYY-MM-DD"
+    if date is None:
+        date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        date = datetime.strptime(date, "%Y-%m-%d").date()
+
+    if performance_platform_client.active:
+
+        send_total_sent_notifications_to_performance_platform(bst_date=date)
+        processing_time.send_processing_time_to_performance_platform(bst_date=date)
+
+
+def send_total_sent_notifications_to_performance_platform(bst_date):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
+    start_time = get_london_midnight_in_utc(bst_date)
+
+    email_sent_count = count_dict['email']
+    sms_sent_count = count_dict['sms']
+    letter_sent_count = count_dict['letter']
+
+    current_app.logger.info(
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
     )
 
-    while len(notifications) > 0:
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-        notifications = dao_timeout_notifications(cutoff_time)
+        start_time,
+        'sms',
+        sms_sent_count
+    )
 
-        for notification in notifications:
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            statsd_client.incr(f'timeout-sending.{notification.sent_by}')
+        start_time,
-            check_and_queue_callback_task(notification)
+        'email',
+        email_sent_count
+    )
 
-        current_app.logger.info(
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+        start_time,
+        'letter',
+        letter_sent_count
+    )
 
 
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
+@statsd(namespace="tasks")
 def delete_inbound_sms():
     try:
         start = datetime.utcnow()
@@ -187,6 +224,7 @@ def delete_inbound_sms():
 
 @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
 @cronitor("raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
     still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
 
@@ -198,16 +236,11 @@ def raise_alert_if_letter_notifications_still_sending():
         # Only send alerts in production
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"  # noqa
-
+            zendesk_client.create_ticket(
-            ticket = NotifySupportTicket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
-                email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
+                ticket_type=zendesk_client.TYPE_INCIDENT
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
         else:
             current_app.logger.info(message)
 
@@ -233,11 +266,17 @@ def get_letter_notifications_still_sending_when_they_shouldnt_be():
         func.date(Notification.sent_at) <= expected_sent_date
     )
 
+    if today.isoweekday() in {2, 4}:  # on tue, thu, we only care about first class letters
+        q = q.filter(
+            Notification.postage == 'first'
+        )
+
     return q.count(), expected_sent_date
 
 
 @notify_celery.task(name='raise-alert-if-no-letter-ack-file')
 @cronitor('raise-alert-if-no-letter-ack-file')
+@statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
     # get a list of zip files since yesterday
     zip_file_set = set()
@@ -255,57 +294,33 @@ def letter_raise_alert_if_no_ack_file_for_zip():
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
-
-    message = '\n'.join([
-        "Letter ack file does not contain all zip files sent."
-        "",
-        f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n",  # noqa
-        f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent",  # noqa
-        f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
-        "",
-        f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
-    ])
 
+    message = (
+        "Letter ack file does not contain all zip files sent. "
+        "Missing ack for zip files: {}, "
+        "pdf bucket: {}, subfolder: {}, "
+        "ack bucket: {}"
+    ).format(
+        str(sorted(zip_file_set - ack_file_set)),
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
+        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
+    )
     # strip empty element before comparison
     ack_file_set.discard('')
     zip_file_set.discard('')
 
     if len(zip_file_set - ack_file_set) > 0:
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
                 subject="Letter acknowledge error",
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
+                ticket_type=zendesk_client.TYPE_INCIDENT
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
         current_app.logger.error(message)
 
     if len(ack_file_set - zip_file_set) > 0:
         current_app.logger.info(
             "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
         )
-
-
-@notify_celery.task(name='save-daily-notification-processing-time')
-@cronitor("save-daily-notification-processing-time")
-def save_daily_notification_processing_time(bst_date=None):
-    # bst_date is a string in the format of "YYYY-MM-DD"
-    if bst_date is None:
-        # if a date is not provided, we run against yesterdays data
-        bst_date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        bst_date = datetime.strptime(bst_date, "%Y-%m-%d").date()
-
-    start_time = get_london_midnight_in_utc(bst_date)
-    end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
-    result = dao_get_notifications_processing_time_stats(start_time, end_time)
-    insert_update_processing_time(
-        FactProcessingTime(
-            bst_date=bst_date,
-            messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs
-        )
-    )

app/celery/process_ses_receipts_tasks.py

@@ -3,30 +3,32 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
-from app.clients.email.aws_ses import get_aws_responses
 from app.config import QueueNames
+from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING, EMAIL_TYPE
+
 from app.notifications.notifications_ses_callback import (
-    _check_and_queue_complaint_callback_task,
-    check_and_queue_callback_task,
     determine_notification_bounce_type,
     handle_complaint,
+    _check_and_queue_complaint_callback_task,
+    _check_and_queue_callback_task,
 )
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_ses_results(self, response):
     try:
         ses_message = json.loads(response['Message'])
         notification_type = ses_message['notificationType']
-        bounce_message = None
 
         if notification_type == 'Bounce':
-            notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
@@ -37,7 +39,9 @@ def process_ses_results(self, response):
         reference = ses_message['mail']['messageId']
 
         try:
-            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
+            notification = notifications_dao.dao_get_notification_or_history_by_reference(
+                reference=reference, notification_type=EMAIL_TYPE
+            )
         except NoResultFound:
             message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
@@ -52,9 +56,6 @@ def process_ses_results(self, response):
                 )
             return
 
-        if bounce_message:
-            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
-
         if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
             notifications_dao._duplicate_update_warning(
                 notification=notification,
@@ -70,13 +71,9 @@ def process_ses_results(self, response):
         statsd_client.incr('callback.ses.{}'.format(notification_status))
 
         if notification.sent_at:
-            statsd_client.timing_with_dates(
+            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
-                f'callback.ses.{notification_status}.elapsed-time',
-                datetime.utcnow(),
-                notification.sent_at
-            )
 
-        check_and_queue_callback_task(notification)
+        _check_and_queue_callback_task(notification)
 
         return True
 

app/celery/process_sms_client_response_tasks.py

@@ -1,27 +1,29 @@
 import uuid
 from datetime import datetime
 
-import pytest
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
 from app.clients import ClientException
+from app.clients.sms.firetext import get_firetext_responses
+from app.clients.sms.mmg import get_mmg_responses
+from app.celery.service_callback_tasks import send_delivery_status_to_service, create_delivery_status_callback_data
+from app.config import QueueNames
 from app.dao import notifications_dao
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 
-# sms_response_mapper = {
+sms_response_mapper = {
-#     'MMG': get_mmg_responses,
+    'MMG': get_mmg_responses,
-#     'Firetext': get_firetext_responses,
+    'Firetext': get_firetext_responses
-# }
+}
 
 
-gUpdate with new providers")
 @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
     # validate reference
     try:
@@ -70,7 +72,7 @@ def _process_for_status(notification_status, client_name, provider_reference, de
 
     if notification.sent_at:
         statsd_client.timing_with_dates(
-            f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
+            'callback.{}.elapsed-time'.format(client_name.lower()),
             datetime.utcnow(),
             notification.sent_at
         )
@@ -89,4 +91,9 @@ def _process_for_status(notification_status, client_name, provider_reference, de
         notifications_dao.dao_update_notification(notification)
 
     if notification_status != NOTIFICATION_PENDING:
-        check_and_queue_callback_task(notification)
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        # queue callback task only if the service_callback_api exists
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)

app/celery/provider_tasks.py

@@ -1,10 +1,9 @@
 from flask import current_app
+from notifications_utils.recipients import InvalidEmailError
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
-from app.clients.email import EmailClientNonRetryableException
-from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
-from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
@@ -14,6 +13,7 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE
 
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
     try:
         current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -21,18 +21,11 @@ def deliver_sms(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_sms_to_provider(notification)
-    except Exception as e:
+    except Exception:
-        if isinstance(e, SmsClientResponseException):
+        try:
-            current_app.logger.warning(
-                "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True
-            )
-        else:
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
-
-        try:
             if self.request.retries == 0:
                 self.retry(queue=QueueNames.RETRY, countdown=0)
             else:
@@ -45,6 +38,7 @@ def deliver_sms(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
     try:
         current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
@@ -52,22 +46,14 @@ def deliver_email(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_email_to_provider(notification)
-    except EmailClientNonRetryableException as e:
+    except InvalidEmailError as e:
-        current_app.logger.exception(
+        current_app.logger.exception(e)
-            f"Email notification {notification_id} failed: {e}"
-        )
         update_notification_status_by_id(notification_id, 'technical-failure')
-    except Exception as e:
+    except Exception:
         try:
-            if isinstance(e, AwsSesClientThrottlingSendRateException):
+            current_app.logger.exception(
-                current_app.logger.warning(
+                "RETRY: Email notification {} failed".format(notification_id)
-                    f"RETRY: Email notification {notification_id} was rate limited by SES"
+            )
-                )
-            else:
-                current_app.logger.exception(
-                    f"RETRY: Email notification {notification_id} failed"
-                )
-
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
             message = "RETRY FAILED: Max retries reached. " \

app/celery/reporting_tasks.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import notify_celery
@@ -8,16 +9,21 @@ from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_billing_dao import (
     fetch_billing_data_for_day,
-    update_fact_billing,
+    update_fact_billing
+)
+from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    LETTER_TYPE,
 )
-from app.dao.fact_notification_status_dao import update_fact_notification_status
-from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE
 
 
 @notify_celery.task(name="create-nightly-billing")
 @cronitor("create-nightly-billing")
+@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
+    current_app.logger.info("create-nightly-billing task: started")
     # day_start is a datetime.date() object. e.g.
     # up to 4 days of data counting back from day_start is consolidated
     if day_start is None:
@@ -25,7 +31,7 @@ def create_nightly_billing(day_start=None):
     else:
         # When calling the task its a string in the format of "YYYY-MM-DD"
         day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 10):
+    for i in range(0, 4):
         process_day = (day_start - timedelta(days=i)).isoformat()
 
         create_nightly_billing_for_day.apply_async(
@@ -38,6 +44,7 @@ def create_nightly_billing(day_start=None):
 
 
 @notify_celery.task(name="create-nightly-billing-for-day")
+@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
     current_app.logger.info(
@@ -63,67 +70,55 @@ def create_nightly_billing_for_day(process_day):
 
 @notify_celery.task(name="create-nightly-notification-status")
 @cronitor("create-nightly-notification-status")
+@statsd(namespace="tasks")
 def create_nightly_notification_status():
-    """
+    current_app.logger.info("create-nightly-notification-status task: started")
-    Aggregate notification statuses into rows in ft_notification_status.
-    In order to minimise effort, this task assumes that:
-
-      - Email + SMS statuses don't change after 3 days. This is currently true
-        because all outstanding email / SMS are "timed out" after 3 days, and
-        we reject delivery receipts after this point.
-
-      - Letter statuses don't change after 9 days. There's no "timeout" for
-        letters but this is the longest we've had to cope with in the past - due
-        to major issues with our print provider.
-
-    Because the time range of the task exceeds the minimum possible retention
-    period (3 days), we need to choose which table to query for each service.
-
-    The aggregation happens for 1 extra day in case:
-
-      - This task or the "timeout" task fails to run.
-
-      - Data is (somehow) still in transit to the history table, which would
-        mean the aggregated results are temporarily incorrect.
-    """
-
     yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
 
-    for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
+    # email and sms
-        days = 10 if notification_type == LETTER_TYPE else 4
+    for i in range(4):
-
+        process_day = yesterday - timedelta(days=i)
-        for i in range(days):
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
-            process_day = yesterday - timedelta(days=i)
+            create_nightly_notification_status_for_day.apply_async(
-
+                kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
-            relevant_service_ids = get_service_ids_with_notifications_on_date(
+                queue=QueueNames.REPORTING
-                notification_type, process_day
             )
-
+            current_app.logger.info(
-            for service_id in relevant_service_ids:
+                f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
-                create_nightly_notification_status_for_service_and_day.apply_async(
+                f"for type {notification_type} for {process_day}"
-                    kwargs={
+            )
-                        'process_day': process_day.isoformat(),
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
-                        'notification_type': notification_type,
+    for i in range(10):
-                        'service_id': service_id,
+        process_day = yesterday - timedelta(days=i)
-                    },
+        create_nightly_notification_status_for_day.apply_async(
-                    queue=QueueNames.REPORTING
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
-                )
+            queue=QueueNames.REPORTING
+        )
+        current_app.logger.info(
+            f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
+            f"for type letter for {process_day}"
+        )
 
 
-@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
+@notify_celery.task(name="create-nightly-notification-status-for-day")
-def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
+@statsd(namespace="tasks")
+def create_nightly_notification_status_for_day(process_day, notification_type):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: started'
+    )
 
     start = datetime.utcnow()
-    update_fact_notification_status(
+    transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
-        process_day=process_day,
-        notification_type=notification_type,
-        service_id=service_id
-    )
-
     end = datetime.utcnow()
     current_app.logger.info(
-        f'create-nightly-notification-status-for-service-and-day task update '
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
-        f'for {service_id}, {notification_type} for {process_day}: '
+        f'data fetched in {(end - start).seconds} seconds'
-        f'updated in {(end - start).seconds} seconds'
+    )
+
+    update_fact_notification_status(transit_data, process_day, notification_type)
+
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
+        f'task complete - {len(transit_data)} rows updated'
     )

app/celery/research_mode_tasks.py

@@ -1,16 +1,17 @@
-import json
 import random
 from datetime import datetime, timedelta
+import json
 
 from flask import current_app
+from requests import request, RequestException, HTTPError
+
 from notifications_utils.s3 import s3upload
-from requests import HTTPError, request
 
 from app import notify_celery
 from app.aws.s3 import file_exists
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
 from app.models import SMS_TYPE
+from app.config import QueueNames
+from app.celery.process_ses_receipts_tasks import process_ses_results
 
 temp_fail = "7700900003"
 perm_fail = "7700900002"
@@ -65,14 +66,16 @@ def make_request(notification_type, provider, data, headers):
             timeout=60
         )
         response.raise_for_status()
-    except HTTPError as e:
+    except RequestException as e:
+        api_error = HTTPError(e)
         current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
+            "API {} request on {} failed with {}".format(
+                "POST",
                 api_call,
-                e.response.status_code
+                api_error.response
             )
         )
-        raise e
+        raise api_error
     finally:
         current_app.logger.info("Mocked provider callback request finished")
     return response.json()
@@ -164,7 +167,7 @@ def ses_notification_callback(reference):
             'processingTimeMillis': 2003,
             'recipients': ['success@simulator.amazonses.com'],
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
             'smtpResponse': '250 2.6.0 Message received',
             'timestamp': '2017-11-17T12:14:03.646Z'
         },
@@ -201,7 +204,7 @@ def ses_notification_callback(reference):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:01.643Z'
         },
@@ -211,14 +214,14 @@ def ses_notification_callback(reference):
     return {
         'Type': 'Notification',
         'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:03.710Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
         'MessageAttributes': {}
     }
 
@@ -244,7 +247,7 @@ def _ses_bounce_callback(reference, bounce_type):
             }],
             'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
             'timestamp': '2017-11-17T12:14:05.131Z'
         },
         'mail': {
@@ -280,7 +283,7 @@ def _ses_bounce_callback(reference, bounce_type):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:03.000Z'
         },
@@ -289,13 +292,13 @@ def _ses_bounce_callback(reference, bounce_type):
     return {
         'Type': 'Notification',
         'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:05.149Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
         'MessageAttributes': {}
     }

app/celery/scheduled_tasks.py

@@ -1,66 +1,55 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
+from notifications_utils.statsd_decorators import statsd
-    NotifySupportTicket,
+from sqlalchemy import and_
-)
-from sqlalchemy import between
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import db, notify_celery, zendesk_client
+from app import notify_celery, zendesk_client
-from app.aws import s3
-from app.celery.broadcast_message_tasks import trigger_link_test
-from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
 from app.celery.tasks import (
-    get_recipient_csv_and_template_and_sender_id,
-    process_incomplete_jobs,
     process_job,
-    process_row,
+    get_recipient_csv_and_template_and_sender_id,
+    process_row
 )
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
 from app.config import QueueNames, TaskNames
-from app.dao.invited_org_user_dao import (
+from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
-    delete_org_invitations_created_more_than_two_days_ago,
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
-)
-from app.dao.invited_user_dao import (
-    delete_invitations_created_more_than_two_days_ago,
-)
 from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
     find_jobs_with_missing_rows,
-    find_missing_row_for_job,
+    find_missing_row_for_job
 )
+from app.dao.jobs_dao import dao_update_job
 from app.dao.notifications_dao import (
-    dao_old_letters_with_created_status,
-    dao_precompiled_letters_still_pending_virus_check,
-    is_delivery_slow_for_providers,
-    letters_missing_from_sending_bucket,
     notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket,
+    is_delivery_slow_for_providers,
 )
 from app.dao.provider_details_dao import (
-    dao_adjust_provider_priority_back_to_resting_points,
     dao_reduce_sms_provider_priority,
-)
+    dao_adjust_provider_priority_back_to_resting_points
-from app.dao.services_dao import (
-    dao_find_services_sending_to_tv_numbers,
-    dao_find_services_with_high_failure_rates,
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.letters.utils import generate_letter_pdf_filename
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
 from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
-    BroadcastMessage,
-    BroadcastStatusType,
     Job,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_ERROR,
+    SMS_TYPE,
+    EMAIL_TYPE,
 )
 from app.notifications.process_notifications import send_notification_to_queue
+from app.v2.errors import JobIncompleteError
 
 
 @notify_celery.task(name="run-scheduled-jobs")
+@statsd(namespace="tasks")
 def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
@@ -72,6 +61,7 @@ def run_scheduled_jobs():
 
 
 @notify_celery.task(name="delete-verify-codes")
+@statsd(namespace="tasks")
 def delete_verify_codes():
     try:
         start = datetime.utcnow()
@@ -85,6 +75,7 @@ def delete_verify_codes():
 
 
 @notify_celery.task(name="delete-invitations")
+@statsd(namespace="tasks")
 def delete_invitations():
     try:
         start = datetime.utcnow()
@@ -99,6 +90,7 @@ def delete_invitations():
 
 
 @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
 def switch_current_sms_provider_on_slow_delivery():
     """
     Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
@@ -121,42 +113,32 @@ def switch_current_sms_provider_on_slow_delivery():
 
 
 @notify_celery.task(name='tend-providers-back-to-middle')
+@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
     dao_adjust_provider_priority_back_to_resting_points()
 
 
 @notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
 def check_job_status():
     """
     every x minutes do this check
     select
     from jobs
     where job_status == 'in progress'
-    and processing started between 30 and 35 minutes ago
+    and template_type in ('sms', 'email')
-    OR where the job_status == 'pending'
+    and scheduled_at or created_at is older that 30 minutes.
-    and the job scheduled_for timestamp is between 30 and 35 minutes ago.
     if any results then
-        update the job_status to 'error'
+        raise error
         process the rows in the csv that are missing (in another task) just do the check here.
     """
     thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
     thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
 
-    incomplete_in_progress_jobs = Job.query.filter(
+    jobs_not_complete_after_30_minutes = Job.query.filter(
         Job.job_status == JOB_STATUS_IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
-    )
+    ).order_by(Job.processing_started).all()
-    incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
-        Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-
-    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
-        incomplete_pending_jobs
-    ).order_by(
-        Job.processing_started, Job.scheduled_for
-    ).all()
 
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
@@ -167,14 +149,16 @@ def check_job_status():
         job_ids.append(str(job.id))
 
     if job_ids:
-        current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
+        notify_celery.send_task(
-        process_incomplete_jobs.apply_async(
+            name=TaskNames.PROCESS_INCOMPLETE_JOBS,
-            [job_ids],
+            args=(job_ids,),
             queue=QueueNames.JOBS
         )
+        raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
 
 
 @notify_celery.task(name='replay-created-notifications')
+@statsd(namespace="tasks")
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
     resend_created_notifications_older_than = (60 * 60)
@@ -207,88 +191,66 @@ def replay_created_notifications():
             get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)
 
 
-@notify_celery.task(name='check-if-letters-still-pending-virus-check')
+@notify_celery.task(name='check-precompiled-letter-state')
-def check_if_letters_still_pending_virus_check():
+@statsd(namespace="tasks")
-    letters = []
+def check_precompiled_letter_state():
-
+    letters = dao_precompiled_letters_still_pending_virus_check()
-    for letter in dao_precompiled_letters_still_pending_virus_check():
-        # find letter in the scan bucket
-        filename = generate_letter_pdf_filename(
-            letter.reference,
-            letter.created_at,
-            ignore_folder=True,
-            postage=letter.postage
-        )
-
-        if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename):
-            current_app.logger.warning(
-                f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.'
-            )
-            notify_celery.send_task(
-                name=TaskNames.SCAN_FILE,
-                kwargs={'filename': filename},
-                queue=QueueNames.ANTIVIRUS,
-            )
-        else:
-            letters.append(letter)
 
     if len(letters) > 0:
-        letter_ids = [(str(letter.id), letter.reference) for letter in letters]
+        letter_ids = [str(letter.id) for letter in letters]
 
-        msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes.
+        msg = """{} precompiled letters have been pending-virus-check for over 90 minutes. Follow runbook to resolve:
-            We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off
+            https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-letter-pending-virus-scan-for-90-minutes.
-            again or move them to technical failure.
+            Notifications: {}""".format(len(letters), letter_ids)
 
-            Notifications: {sorted(letter_ids)}"""
+        current_app.logger.exception(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check",
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
                 message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
+                ticket_type=zendesk_client.TYPE_INCIDENT
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)
 
 
-@notify_celery.task(name='check-if-letters-still-in-created')
+@notify_celery.task(name='check-templated-letter-state')
-def check_if_letters_still_in_created():
+@statsd(namespace="tasks")
+def check_templated_letter_state():
     letters = dao_old_letters_with_created_status()
 
     if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
         msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Follow runbook to resolve: " \
+              "Notifications: {}".format(len(letters), letter_ids)
-              "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
+
-              "#deal-with-Letters-still-in-created.".format(len(letters))
+        current_app.logger.exception(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status",
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
                 message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
+                ticket_type=zendesk_client.TYPE_INCIDENT
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)
 
 
 @notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
 def check_for_missing_rows_in_completed_jobs():
-    jobs = find_jobs_with_missing_rows()
+    jobs_and_job_size = find_jobs_with_missing_rows()
-    for job in jobs:
+    for x in jobs_and_job_size:
-        recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
+        job = x[1]
         missing_rows = find_missing_row_for_job(job.id, job.notification_count)
         for row_to_process in missing_rows:
-            row = recipient_csv[row_to_process.missing_row]
+            recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
-            current_app.logger.info(
+            for row in recipient_csv.get_rows():
-                "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
+                if row.index == row_to_process.missing_row:
-            process_row(row, template, job, job.service, sender_id=sender_id)
+                    current_app.logger.info(
+                        "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
+                    process_row(row, template, job, job.service, sender_id=sender_id)
 
 
 @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
+@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
     start_date = (datetime.utcnow() - timedelta(days=1))
     end_date = datetime.utcnow()
@@ -326,44 +288,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             message += ("\nYou can find instructions for this ticket in our manual:\n"
                         "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
+                subject="[{}] High failure rates for sms spotted for services".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']
+                ),
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
+                ticket_type=zendesk_client.TYPE_INCIDENT
-                technical_ticket=True
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-@notify_celery.task(name='trigger-link-tests')
-def trigger_link_tests():
-    if current_app.config['CBC_PROXY_ENABLED']:
-        for cbc_name in current_app.config['ENABLED_CBCS']:
-            trigger_link_test.apply_async(kwargs={'provider': cbc_name}, queue=QueueNames.BROADCASTS)
-
-
-@notify_celery.task(name='auto-expire-broadcast-messages')
-def auto_expire_broadcast_messages():
-    expired_broadcasts = BroadcastMessage.query.filter(
-        BroadcastMessage.finishes_at <= datetime.now(),
-        BroadcastMessage.status == BroadcastStatusType.BROADCASTING,
-    ).all()
-
-    for broadcast in expired_broadcasts:
-        broadcast.status = BroadcastStatusType.COMPLETED
-
-    db.session.commit()
-
-    if expired_broadcasts:
-        notify_celery.send_task(
-            name=TaskNames.PUBLISH_GOVUK_ALERTS,
-            queue=QueueNames.GOVUK_ALERTS
-        )
-
-
-@notify_celery.task(name='remove-yesterdays-planned-tests-on-govuk-alerts')
-def remove_yesterdays_planned_tests_on_govuk_alerts():
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )

app/celery/service_callback_tasks.py

@@ -1,14 +1,22 @@
 import json
 
 from flask import current_app
-from requests import HTTPError, RequestException, request
+from notifications_utils.statsd_decorators import statsd
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
 
-from app import encryption, notify_celery
+from app import (
+    notify_celery,
+    encryption
+)
 from app.config import QueueNames
-from app.utils import DATETIME_FORMAT
 
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
@@ -22,11 +30,8 @@ def send_delivery_status_to_service(
         "created_at": status_update['notification_created_at'],
         "completed_at": status_update['notification_updated_at'],
         "sent_at": status_update['notification_sent_at'],
-        "notification_type": status_update['notification_type'],
+        "notification_type": status_update['notification_type']
-        "template_id": status_update['template_id'],
-        "template_version": status_update['template_version']
     }
-
     _send_data_to_service_callback_api(
         self,
         data,
@@ -37,6 +42,7 @@ def send_delivery_status_to_service(
 
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
     complaint = encryption.decrypt(complaint_data)
 
@@ -68,7 +74,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 'Content-Type': 'application/json',
                 'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=5
+            timeout=60
         )
         current_app.logger.info('{} sending {} to {}, response {}'.format(
             function_name,
@@ -79,16 +85,16 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            "{} request failed for notification_id: {} and url: {}. exception: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
                 function_name,
                 notification_id,
                 service_callback_url,
                 e
             )
         )
-        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
-                self.retry(queue=QueueNames.CALLBACKS_RETRY)
+                self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.warning(
                     "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
@@ -97,18 +103,10 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                         notification_id
                     )
                 )
-        else:
-            current_app.logger.warning(
-                "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name,
-                    notification_id,
-                    service_callback_url,
-                    e
-                )
-            )
 
 
 def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "notification_id": str(notification.id),
         "notification_client_reference": notification.client_reference,
@@ -121,13 +119,12 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_type": notification.notification_type,
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
-        "template_id": str(notification.template_id),
-        "template_version": notification.template_version,
     }
     return encryption.encrypt(data)
 
 
 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "complaint_id": str(complaint.id),
         "notification_id": str(notification.id),

app/celery/tasks.py

@@ -1,38 +1,49 @@
 import json
-from collections import defaultdict, namedtuple
 from datetime import datetime
+from collections import namedtuple, defaultdict
 
 from flask import current_app
-from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.columns import Columns
 from notifications_utils.postal_address import PostalAddress
 from notifications_utils.recipients import RecipientCSV
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
-from requests import HTTPError, RequestException, request
+from requests import (
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+    HTTPError,
+    request,
+    RequestException
+)
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 
-from app import create_random_identifier, create_uuid, encryption, notify_celery
+from app import (
+    create_uuid,
+    create_random_identifier,
+    DATETIME_FORMAT,
+    encryption,
+    notify_celery,
+)
 from app.aws import s3
-from app.celery import letters_pdf_tasks, provider_tasks, research_mode_tasks
+from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
 from app.config import QueueNames
-from app.dao.daily_sorted_letter_dao import (
+from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
-    dao_create_or_update_daily_sorted_letter,
-)
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
-from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id,
+)
 from app.dao.notifications_dao import (
-    dao_get_last_notification_added_for_job_id,
-    dao_get_notification_or_history_by_reference,
-    dao_update_notifications_by_reference,
     get_notification_by_id,
+    dao_update_notifications_by_reference,
+    dao_get_last_notification_added_for_job_id,
     update_notification_status_by_reference,
+    dao_get_notification_or_history_by_reference,
 )
-from app.dao.provider_details_dao import (
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
-    get_provider_details_by_notification_type,
-)
 from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
@@ -46,22 +57,19 @@ from app.models import (
     LETTER_TYPE,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
-    NOTIFICATION_RETURNED_LETTER,
     NOTIFICATION_SENDING,
-    NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
     SMS_TYPE,
     DailySortedLetter,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
-from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
-from app.v2.errors import TooManyRequestsError
 
 
 @notify_celery.task(name="process-job")
+@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
@@ -72,10 +80,6 @@ def process_job(job_id, sender_id=None):
 
     service = job.service
 
-    job.job_status = JOB_STATUS_IN_PROGRESS
-    job.processing_started = start
-    dao_update_job(job)
-
     if not service.active:
         job.job_status = JOB_STATUS_CANCELLED
         dao_update_job(job)
@@ -86,6 +90,10 @@ def process_job(job_id, sender_id=None):
     if __sending_limits_for_job_exceeded(service, job, job_id):
         return
 
+    job.job_status = JOB_STATUS_IN_PROGRESS
+    job.processing_started = start
+    dao_update_job(job)
+
     recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
 
     current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
@@ -160,13 +168,9 @@ def process_row(row, template, job, service, sender_id=None):
 
 
 def __sending_limits_for_job_exceeded(service, job, job_id):
-    try:
+    total_sent = fetch_todays_total_message_count(service.id)
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
+
-        if total_sent + job.notification_count > service.message_limit:
+    if total_sent + job.notification_count > service.message_limit:
-            raise TooManyRequestsError(service.message_limit)
-        else:
-            return False
-    except TooManyRequestsError:
         job.job_status = 'sending limits exceeded'
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
@@ -175,26 +179,24 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
                 job_id, job.notification_count, service.message_limit)
         )
         return True
+    return False
 
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_sms(self,
              service_id,
              notification_id,
              encrypted_notification,
              sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(service_id)
+    service = dao_fetch_service_by_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
 
     if sender_id:
         reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.debug(
@@ -236,6 +238,7 @@ def save_sms(self,
 
 
 @notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_email(self,
                service_id,
                notification_id,
@@ -243,17 +246,13 @@ def save_email(self,
                sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
 
-    service = SerialisedService.from_id(service_id)
+    service = dao_fetch_service_by_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
 
     if sender_id:
         reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.info("Email {} failed as restricted service".format(notification_id))
@@ -287,22 +286,14 @@ def save_email(self,
 
 
 @notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
-def save_api_email(self, encrypted_notification):
+@statsd(namespace="tasks")
+def save_api_email(self,
+                   encrypted_notification,
+                   ):
 
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
-def save_api_sms(self, encrypted_notification):
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-def save_api_email_or_sms(self, encrypted_notification):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification['service_id'])
+    service = dao_fetch_service_by_id(notification['service_id'])
-    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
+
-    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
-        else provider_tasks.deliver_sms
     try:
 
         persist_notification(
@@ -312,7 +303,7 @@ def save_api_email_or_sms(self, encrypted_notification):
             recipient=notification['to'],
             service=service,
             personalisation=notification.get('personalisation'),
-            notification_type=notification['notification_type'],
+            notification_type=EMAIL_TYPE,
             client_reference=notification['client_reference'],
             api_key_id=notification.get('api_key_id'),
             key_type=KEY_TYPE_NORMAL,
@@ -322,26 +313,25 @@ def save_api_email_or_sms(self, encrypted_notification):
             document_download_count=notification['document_download_count']
         )
 
-        q = q if not service.research_mode else QueueNames.RESEARCH_MODE
+        q = QueueNames.SEND_EMAIL if not service.research_mode else QueueNames.RESEARCH_MODE
-        provider_task.apply_async(
+        provider_tasks.deliver_email.apply_async(
             [notification['id']],
             queue=q
         )
-        current_app.logger.debug(
+        current_app.logger.info(f"Email {notification['id']} has been persisted and sent to delivery queue.")
-            f"{notification['notification_type']} {notification['id']} has been persisted and sent to delivery queue."
-        )
     except IntegrityError:
-        current_app.logger.info(f"{notification['notification_type']} {notification['id']} already exists.")
+        current_app.logger.info(f"Email {notification['id']} already exists.")
 
     except SQLAlchemyError:
 
         try:
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
-            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
+            current_app.logger.error('Max retry failed' + f"Failed to persist notification {notification['id']}")
 
 
 @notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_letter(
         self,
         service_id,
@@ -350,16 +340,12 @@ def save_letter(
 ):
     notification = encryption.decrypt(encrypted_notification)
 
-    postal_address = PostalAddress.from_personalisation(
+    recipient = PostalAddress.from_personalisation(
-        InsensitiveDict(notification['personalisation'])
+        Columns(notification['personalisation'])
-    )
+    ).normalised
 
-    service = SerialisedService.from_id(service_id)
+    service = dao_fetch_service_by_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
 
     try:
         # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
@@ -368,8 +354,8 @@ def save_letter(
         saved_notification = persist_notification(
             template_id=notification['template'],
             template_version=notification['template_version'],
-            postage=postal_address.postage if postal_address.international else template.postage,
+            template_postage=template.postage,
-            recipient=postal_address.normalised,
+            recipient=recipient,
             service=service,
             personalisation=notification['personalisation'],
             notification_type=LETTER_TYPE,
@@ -380,8 +366,7 @@ def save_letter(
             job_row_number=notification['row_number'],
             notification_id=notification_id,
             reference=create_random_identifier(),
-            client_reference=get_reference_from_personalisation(notification['personalisation']),
+            reply_to_text=template.get_reply_to_text(),
-            reply_to_text=template.reply_to_text,
             status=status
         )
 
@@ -404,6 +389,7 @@ def save_letter(
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
     provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
@@ -422,6 +408,7 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
 
@@ -457,6 +444,7 @@ def handle_exception(task, notification, notification_id, exc):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
 def update_letter_notifications_statuses(self, filename):
     notification_updates = parse_dvla_file(filename)
 
@@ -473,6 +461,7 @@ def update_letter_notifications_statuses(self, filename):
 
 
 @notify_celery.task(bind=True, name="record-daily-sorted-counts")
+@statsd(namespace="tasks")
 def record_daily_sorted_counts(self, filename):
     sorted_letter_counts = defaultdict(int)
     notification_updates = parse_dvla_file(filename)
@@ -547,7 +536,7 @@ def update_letter_notification(filename, temporary_failures, update):
 
 
 def check_billable_units(notification_update):
-    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_or_history_by_reference(notification_update.reference, LETTER_TYPE)
 
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(
@@ -559,6 +548,7 @@ def check_billable_units(notification_update):
 
 
 @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
     inbound_api = get_service_inbound_api_for_service(service_id=service_id)
     if not inbound_api:
@@ -587,32 +577,36 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
             },
             timeout=60
         )
-        current_app.logger.debug(
+        current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
+            inbound_sms_id,
-            f"response {response.status_code}"
+            inbound_api.url,
-        )
+            response.status_code
+        ))
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
+            "send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
-            f"and url: {inbound_api.url}. exception: {e}"
+                service_id,
+                inbound_sms_id,
+                inbound_api.url,
+                e
+            )
         )
         if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
                 self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of" +
+                    """Retry: send_inbound_sms_to_service has retried the max number of
-                    f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
+                     times for service: {} and  inbound_sms {}""".format(
+                        service_id,
+                        inbound_sms_id
+                    )
                 )
-        else:
-            current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
-                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
-            )
 
 
 @notify_celery.task(name='process-incomplete-jobs')
+@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
     jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
 
@@ -649,6 +643,7 @@ def process_incomplete_job(job_id):
 
 
 @notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
 def process_returned_letters_list(notification_references):
     updated, updated_history = dao_update_notifications_by_reference(
         notification_references,

app/clients/__init__.py

@@ -1,6 +1,3 @@
-from celery import current_app
-
-
 class ClientException(Exception):
     '''
     Base Exceptions for sending notifications that fail
@@ -20,7 +17,7 @@ STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'
 
 
-class NotificationProviderClients(object):
+class Clients(object):
     sms_clients = {}
     email_clients = {}
 
@@ -39,7 +36,7 @@ class NotificationProviderClients(object):
 
     def get_client_by_name_and_type(self, name, notification_type):
         assert notification_type in ['email', 'sms']
-        
+
         if notification_type == 'email':
             return self.get_email_client(name)
 

app/clients/cbc_proxy.py

@@ -1,299 +0,0 @@
-import json
-import uuid
-from abc import ABC, abstractmethod
-
-import boto3
-import botocore
-from flask import current_app
-from notifications_utils.template import non_gsm_characters
-from sqlalchemy.schema import Sequence
-
-from app.config import BroadcastProvider
-from app.utils import DATETIME_FORMAT, format_sequential_number
-
-# The variable names in this file have specific meaning in a CAP message
-#
-# identifier is a unique field for each CAP message
-#
-# headline is a field which we are not sure if we will use
-#
-# description is the body of the message
-
-# areas is a list of dicts, with the following items
-# * description is a string which populates the areaDesc field
-# * polygon is a list of lat/long pairs
-#
-# previous_provider_messages is a list of previous events (models.py::BroadcastProviderMessage)
-# ie a Cancel message would have a unique event but have the event of
-#    the preceeding Alert message in the previous_provider_messages field
-
-
-class CBCProxyRetryableException(Exception):
-    pass
-
-
-class CBCProxyClient:
-    _lambda_client = None
-
-    def init_app(self, app):
-        if app.config.get('CBC_PROXY_ENABLED'):
-            self._lambda_client = boto3.client(
-                'lambda',
-                region_name='us-west-2',
-                aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
-                aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
-            )
-
-    def get_proxy(self, provider):
-        proxy_classes = {
-            BroadcastProvider.EE: CBCProxyEE,
-            BroadcastProvider.THREE: CBCProxyThree,
-            BroadcastProvider.O2: CBCProxyO2,
-            BroadcastProvider.VODAFONE: CBCProxyVodafone,
-        }
-        return proxy_classes[provider](self._lambda_client)
-
-
-class CBCProxyClientBase(ABC):
-    @property
-    @abstractmethod
-    def lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def failover_lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_ENGLISH(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_WELSH(self):
-        pass
-
-    def __init__(self, lambda_client):
-        self._lambda_client = lambda_client
-
-    def send_link_test(self):
-        self._send_link_test(self.lambda_name)
-        self._send_link_test(self.failover_lambda_name)
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ): pass
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        pass
-
-    # We have not implementated updating a broadcast
-    def update_and_send_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, channel, message_number=None
-    ):
-        pass
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, message_number=None
-    ):
-        pass
-
-    def _invoke_lambda_with_failover(self, payload):
-        result = self._invoke_lambda(self.lambda_name, payload)
-
-        if not result:
-            failover_result = self._invoke_lambda(self.failover_lambda_name, payload)
-            if not failover_result:
-                raise CBCProxyRetryableException(
-                    f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}'
-                )
-
-        return result
-
-    def _invoke_lambda(self, lambda_name, payload):
-        payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        try:
-            current_app.logger.info(
-                f"Calling lambda {lambda_name} with payload {str(payload)[:1000]}"
-            )
-
-            result = self._lambda_client.invoke(
-                FunctionName=lambda_name,
-                InvocationType='RequestResponse',
-                Payload=payload_bytes,
-            )
-        except botocore.exceptions.ClientError:
-            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
-            success = False
-            return success
-
-        if result['StatusCode'] > 299:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with status code { result['StatusCode']}, {result.get('Payload')}"
-            )
-            success = False
-
-        elif 'FunctionError' in result:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with function error { result['Payload'].read() }"
-            )
-            success = False
-
-        else:
-            success = True
-
-        return success
-
-    def infer_language_from(self, content):
-        if non_gsm_characters(content):
-            return self.LANGUAGE_WELSH
-        return self.LANGUAGE_ENGLISH
-
-
-class CBCProxyOne2ManyClient(CBCProxyClientBase):
-    LANGUAGE_ENGLISH = 'en-GB'
-    LANGUAGE_WELSH = 'cy-GB'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_format': 'cap'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_format': 'cap',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages,
-        sent, message_number=None
-    ):
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_format': 'cap',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-
-class CBCProxyEE(CBCProxyOne2ManyClient):
-    lambda_name = 'ee-1-proxy'
-    failover_lambda_name = 'ee-2-proxy'
-
-
-class CBCProxyThree(CBCProxyOne2ManyClient):
-    lambda_name = 'three-1-proxy'
-    failover_lambda_name = 'three-2-proxy'
-
-
-class CBCProxyO2(CBCProxyOne2ManyClient):
-    lambda_name = 'o2-1-proxy'
-    failover_lambda_name = 'o2-2-proxy'
-
-
-class CBCProxyVodafone(CBCProxyClientBase):
-    lambda_name = 'vodafone-1-proxy'
-    failover_lambda_name = 'vodafone-2-proxy'
-
-    LANGUAGE_ENGLISH = 'English'
-    LANGUAGE_WELSH = 'Welsh'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        from app import db
-        sequence = Sequence('broadcast_provider_message_number_seq')
-        sequential_number = db.session.connection().execute(sequence)
-        formatted_seq_number = format_sequential_number(sequential_number)
-
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_number': formatted_seq_number,
-            'message_format': 'ibag'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, message_number, headline, description, areas, sent, expires, channel
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self, identifier, previous_provider_messages, sent, message_number
-    ):
-
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "message_number": format_sequential_number(message.message_number),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)

app/clients/document_download.py

@@ -1,4 +1,5 @@
 import requests
+
 from flask import current_app
 
 
@@ -41,9 +42,7 @@ class DocumentDownloadClient:
             # if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
             # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
             # unrecognised) should be raised as a 500 internal server error here.
-            if e.response is None:
+            if e.response.status_code == 400:
-                raise Exception(f'Unhandled document download error: {repr(e)}')
-            elif e.response.status_code == 400:
                 error = DocumentDownloadError.from_exception(e)
                 current_app.logger.info(
                     'Document download request failed with error: {}'.format(error.message)

app/clients/email/__init__.py

@@ -1,4 +1,4 @@
-from app.clients import Client, ClientException
+from app.clients import ClientException, Client
 
 
 class EmailClientException(ClientException):
@@ -8,18 +8,6 @@ class EmailClientException(ClientException):
     pass
 
 
-class EmailClientNonRetryableException(ClientException):
-    '''
-    Represents an error returned from the email client API with a 4xx response code
-    that should not be retried and should instead be marked as technical failure.
-    An example of this would be an email address that makes it through our
-    validation rules but is rejected by SES. There is no point in retrying this type as
-    it will always fail however many calls to SES. Whereas a throttling error would not
-    use this exception as it may succeed if we retry
-    '''
-    pass
-
-
 class EmailClient(Client):
     '''
     Base Email client for sending emails.
@@ -28,6 +16,5 @@ class EmailClient(Client):
     def send_email(self, *args, **kwargs):
         raise NotImplementedError('TODO Need to implement.')
 
-    @property
+    def get_name(self):
-    def name(self):
         raise NotImplementedError('TODO Need to implement.')

app/clients/email/aws_ses.py

@@ -1,15 +1,11 @@
-from time import monotonic
-
 import boto3
 import botocore
 from flask import current_app
+from time import monotonic
+from notifications_utils.recipients import InvalidEmailError
 
 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
-from app.clients.email import (
+from app.clients.email import (EmailClientException, EmailClient)
-    EmailClient,
-    EmailClientException,
-    EmailClientNonRetryableException,
-)
 
 ses_response_map = {
     'Permanent': {
@@ -47,10 +43,6 @@ class AwsSesClientException(EmailClientException):
     pass
 
 
-class AwsSesClientThrottlingSendRateException(AwsSesClientException):
-    pass
-
-
 class AwsSesClient(EmailClient):
     '''
     Amazon SES email client.
@@ -59,11 +51,27 @@ class AwsSesClient(EmailClient):
     def init_app(self, region, statsd_client, *args, **kwargs):
         self._client = boto3.client('ses', region_name=region)
         super(AwsSesClient, self).__init__(*args, **kwargs)
+        self.name = 'ses'
         self.statsd_client = statsd_client
 
-    @property
+        # events are generally undocumented, but some that might be of interest are:
-    def name(self):
+        # before-call, after-call, after-call-error, request-created, response-received
-        return 'ses'
+        self._client.meta.events.register(f'request-created.ses.SendEmail', self.ses_request_created_hook)
+        self._client.meta.events.register(f'response-received.ses.SendEmail', self.ses_response_received_hook)
+
+    def ses_request_created_hook(self, **kwargs):
+        # request created may be called multiple times if the request auto-retries. We want to count all these as the
+        # same request for timing purposes, so only reset the start time if it was cleared completely
+        if self.ses_start_time == 0:
+            self.ses_start_time = monotonic()
+
+    def ses_response_received_hook(self, **kwargs):
+        # response received may be called multiple times if the request auto-retries, however, we want to count the last
+        # time it triggers for timing purposes, so always reset the elapsed time
+        self.ses_elapsed_time = monotonic() - self.ses_start_time
+
+    def get_name(self):
+        return self.name
 
     def send_email(self,
                    source,
@@ -72,6 +80,8 @@ class AwsSesClient(EmailClient):
                    body,
                    html_body='',
                    reply_to_address=None):
+        self.ses_elapsed_time = 0
+        self.ses_start_time = 0
         try:
             if isinstance(to_addresses, str):
                 to_addresses = [to_addresses]
@@ -108,12 +118,10 @@ class AwsSesClient(EmailClient):
 
             # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
             if e.response['Error']['Code'] == 'InvalidParameterValue':
-                raise EmailClientNonRetryableException(e.response['Error']['Message'])
+                raise InvalidEmailError('email: "{}" message: "{}"'.format(
-            elif (
+                    to_addresses[0],
-                e.response['Error']['Code'] == 'Throttling'
+                    e.response['Error']['Message']
-                and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
+                ))
-            ):
-                raise AwsSesClientThrottlingSendRateException(str(e))
             else:
                 self.statsd_client.incr("clients.ses.error")
                 raise AwsSesClientException(str(e))
@@ -124,6 +132,8 @@ class AwsSesClient(EmailClient):
             elapsed_time = monotonic() - start_time
             current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
             self.statsd_client.timing("clients.ses.request-time", elapsed_time)
+            if self.ses_elapsed_time != 0:
+                self.statsd_client.timing("clients.ses.raw-request-time", self.ses_elapsed_time)
             self.statsd_client.incr("clients.ses.success")
             return response['MessageId']
 

app/clients/email/aws_ses_stub.py

@@ -1,10 +1,10 @@
 import json
-from time import monotonic
 
 from flask import current_app
 from requests import request
+from time import monotonic
 
-from app.clients.email import EmailClient, EmailClientException
+from app.clients.email import (EmailClientException, EmailClient)
 
 
 class AwsSesStubClientException(EmailClientException):
@@ -13,12 +13,12 @@ class AwsSesStubClientException(EmailClientException):
 
 class AwsSesStubClient(EmailClient):
     def init_app(self, region, statsd_client, stub_url):
+        self.name = 'ses'
         self.statsd_client = statsd_client
         self.url = stub_url
 
-    @property
+    def get_name(self):
-    def name(self):
+        return self.name
-        return 'ses'
 
     def send_email(self,
                    source,

app/clients/performance_platform/performance_platform_client.py

@@ -1,8 +1,9 @@
 import base64
 import json
 
-import requests
 from flask import current_app
+import requests
+
 from notifications_utils.timezones import convert_utc_to_bst
 
 

app/clients/sms/__init__.py

@@ -1,10 +1,10 @@
-from app.clients import Client, ClientException
+from app.clients import (Client, ClientException)
 
 
 class SmsClientResponseException(ClientException):
-    """
+    '''
     Base Exception for SmsClientsResponses
-    """
+    '''
 
     def __init__(self, message):
         self.message = message
@@ -14,15 +14,12 @@ class SmsClientResponseException(ClientException):
 
 
 class SmsClient(Client):
-    """
+    '''
     Base Sms client for sending smss.
-    """
+    '''
-
-    def init_app(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
 
     def send_sms(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
 
     def get_name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

app/clients/sms/aws_sns.py

@@ -1,89 +0,0 @@
-import re
-from time import monotonic
-
-import boto3
-import botocore
-import phonenumbers
-
-from app.clients.sms import SmsClient
-
-
-class AwsSnsClient(SmsClient):
-    """
-    AwsSns sms client
-    """
-
-    def init_app(self, current_app, statsd_client, *args, **kwargs):
-        self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"])
-        self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"])
-        super(SmsClient, self).__init__(*args, **kwargs)
-        self.current_app = current_app
-        self.statsd_client = statsd_client
-        self.long_code_regex = re.compile(r"^\+1\d{10}$")
-        
-    @property
-    def name(self):
-        return 'sns'
-
-    def get_name(self):
-        return 'sns'
-
-    def send_sms(self, to, content, reference, sender=None, international=False):
-        matched = False
-
-        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
-            matched = True
-            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
-
-            client = self._client
-            # See documentation
-            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
-            attributes = {
-                "AWS.SNS.SMS.SMSType": {
-                    "DataType": "String",
-                    "StringValue": "Transactional",
-                }
-            }
-
-            # If sending with a long code number, we need to use another AWS region
-            # and specify the phone number we want to use as the origination number
-            send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender)
-            if send_with_dedicated_phone_number:
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": sender,
-                }
-
-            # If the number is US based, we must use a US Toll Free number to send the message
-            country = phonenumbers.region_code_for_number(match.number)
-            if country == "US":
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
-                }
-
-            try:
-                start_time = monotonic()
-                response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
-            except botocore.exceptions.ClientError as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            except Exception as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            finally:
-                elapsed_time = monotonic() - start_time
-                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
-                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
-                self.statsd_client.incr("clients.sns.success")
-            return response["MessageId"]
-
-        if not matched:
-            self.statsd_client.incr("clients.sns.error")
-            self.current_app.logger.error("No valid numbers found in {}".format(to))
-            raise ValueError("No valid numbers found for SMS delivery")
-
-    def _send_with_dedicated_phone_number(self, sender):
-        return sender and re.match(self.long_code_regex, sender)

app/clients/sms/firetext.py

@@ -0,0 +1,130 @@
+import json
+import logging
+
+from time import monotonic
+from requests import request, RequestException
+
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+logger = logging.getLogger(__name__)
+
+# Firetext will send a delivery receipt with three different status codes.
+# The `firetext_response` maps these codes to the notification statistics status and notification status.
+# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
+# the notification status to temporary-failure rather than permanent failure.
+#  See the code in the notification_dao.update_notifications_status_by_id
+firetext_responses = {
+    '0': 'delivered',
+    '1': 'permanent-failure',
+    '2': 'pending'
+}
+
+firetext_codes = {
+    # code '000' means 'No errors reported'
+    '101': {'status': 'permanent-failure', 'reason': 'Unknown Subscriber'},
+    '102': {'status': 'temporary-failure', 'reason': 'Absent Subscriber'},
+    '103': {'status': 'temporary-failure', 'reason': 'Subscriber Busy'},
+    '104': {'status': 'temporary-failure', 'reason': 'No Subscriber Memory'},
+    '201': {'status': 'permanent-failure', 'reason': 'Invalid Number'},
+    '301': {'status': 'permanent-failure', 'reason': 'SMS Not Supported'},
+    '302': {'status': 'temporary-failure', 'reason': 'SMS Not Supported'},
+    '401': {'status': 'permanent-failure', 'reason': 'Message Rejected'},
+    '900': {'status': 'temporary-failure', 'reason': 'Routing Error'},
+}
+
+
+def get_firetext_responses(status, detailed_status_code=None):
+    detailed_status = firetext_codes[detailed_status_code]['reason'] if firetext_codes.get(
+        detailed_status_code, None
+    ) else None
+    return (firetext_responses[status], detailed_status)
+
+
+def get_message_status_and_reason_from_firetext_code(detailed_status_code):
+    return firetext_codes[detailed_status_code]['status'], firetext_codes[detailed_status_code]['reason']
+
+
+class FiretextClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class FiretextClient(SmsClient):
+    '''
+    FireText sms client.
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('FIRETEXT_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'firetext'
+        self.url = current_app.config.get('FIRETEXT_URL')
+        self.statsd_client = statsd_client
+
+    def get_name(self):
+        return self.name
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.firetext.success")
+        else:
+            self.statsd_client.incr("clients.firetext.error")
+            self.current_app.logger.error(log_message)
+
+    def send_sms(self, to, content, reference, sender=None):
+
+        data = {
+            "apiKey": self.api_key,
+            "from": self.from_number if sender is None else sender,
+            "to": to.replace('+', ''),
+            "message": content,
+            "reference": reference
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.url,
+                data=data,
+                timeout=60
+            )
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+                if response.json()['code'] != 0:
+                    raise ValueError()
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise FiretextClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise FiretextClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
+            self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.firetext.raw-request-time", response.elapsed.total_seconds())
+        return response

app/clients/sms/mmg.py

@@ -0,0 +1,140 @@
+import json
+from time import monotonic
+from requests import (request, RequestException)
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+mmg_response_map = {
+    '2': {'status': 'permanent-failure', 'substatus': {
+        "1": "Number does not exist",
+        "4": "Rejected by operator",
+        "5": "Unidentified Subscriber",
+        "9": "Undelivered",
+        "11": "Service for Subscriber suspended",
+        "12": "Illegal equipment",
+        "2049": "Subscriber IMSI blacklisted",
+        "2050": "Number blacklisted in do-not-disturb blacklist",
+        "2052": "Destination number blacklisted",
+        "2053": "Source address blacklisted"
+    }},
+    '3': {'status': 'delivered', 'substatus': {"2": "Delivered to operator", "5": "Delivered to handset"}},
+    '4': {'status': 'temporary-failure', 'substatus': {
+        "6": "Absent Subscriber",
+        "8": "Roaming not allowed",
+        "13": "SMS Not Supported",
+        "15": "Expired",
+        "27": "Absent Subscriber",
+        "29": "Invalid delivery report",
+        "32": "Delivery Failure",
+    }},
+    '5': {'status': 'permanent-failure', 'substatus': {
+        "6": "Network out of coverage",
+        "8": "Incorrect number prefix",
+        "10": "Number on do-not-disturb service",
+        "11": "Sender id not registered",
+        "13": "Sender id blacklisted",
+        "14": "Destination number blacklisted",
+        "19": "Routing unavailable",
+        "20": "Rejected by anti-flooding mechanism",
+        "21": "System error",  # it says to retry those messages or contact support
+        "23": "Duplicate message id",
+        "24": "Message formatted incorrectly",
+        "25": "Message too long",
+        "51": "Missing recipient value",
+        "52": "Invalid destination",
+    }},
+}
+
+
+def get_mmg_responses(status, detailed_status_code=None):
+    return (mmg_response_map[status]["status"], mmg_response_map[status]["substatus"].get(detailed_status_code, None))
+
+
+class MMGClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class MMGClient(SmsClient):
+    '''
+    MMG sms client
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('MMG_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'mmg'
+        self.statsd_client = statsd_client
+        self.mmg_url = current_app.config.get('MMG_URL')
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.mmg_url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.mmg.success")
+        else:
+            self.statsd_client.incr("clients.mmg.error")
+            self.current_app.logger.error(log_message)
+
+    def get_name(self):
+        return self.name
+
+    def send_sms(self, to, content, reference, multi=True, sender=None):
+        data = {
+            "reqType": "BULK",
+            "MSISDN": to,
+            "msg": content,
+            "sender": self.from_number if sender is None else sender,
+            "cid": reference,
+            "multi": multi
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.mmg_url,
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Basic {}'.format(self.api_key)
+                },
+                timeout=60
+            )
+
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise MMGClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise MMGClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.mmg.raw-request-time", response.elapsed.total_seconds())
+
+            self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
+
+        return response

app/cloudfoundry_config.py

@@ -1,12 +1,21 @@
-import json
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
 import os
+import json
 
 
 def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+    set_config_env_vars(vcap_services)
 
+
+def set_config_env_vars(vcap_services):
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']
-                                                                                                      'postgresql')
+
-    # Redis config
+    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri']
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'

app/commands.py

@@ -1,33 +1,29 @@
 import csv
 import functools
-import itertools
-import os
 import uuid
 from datetime import datetime, timedelta
+from decimal import Decimal
 
 import click
 import flask
+import itertools
 from click_datetime import Datetime as click_dt
 from flask import current_app, json
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
-from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from notifications_utils.statsd_decorators import statsd
 
-from app import db
+from app import db, DATETIME_FORMAT, encryption
 from app.aws import s3
-from app.celery.letters_pdf_tasks import (
+from app.celery.tasks import record_daily_sorted_counts, process_row
-    get_pdf_for_templated_letter,
+from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
-    resanitise_pdf,
+from app.celery.service_callback_tasks import send_delivery_status_to_service
-)
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
-from app.celery.tasks import process_row, record_daily_sorted_counts
+from app.celery.reporting_tasks import create_nightly_notification_status_for_day
 from app.config import QueueNames
-from app.dao.annual_billing_dao import (
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
-    dao_create_or_update_annual_billing_for_year,
-    set_default_free_allowance_for_service,
-)
 from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
@@ -35,40 +31,36 @@ from app.dao.fact_billing_dao import (
     update_fact_billing,
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.dao.organisation_dao import (
+from app.dao.organisation_dao import dao_get_organisation_by_email_address, dao_add_service_to_organisation
-    dao_add_service_to_organisation,
+
-    dao_get_organisation_by_email_address,
+from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
-    dao_get_organisation_by_id,
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
-)
-from app.dao.permissions_dao import permission_dao
 from app.dao.services_dao import (
+    delete_service_and_all_associated_db_objects,
     dao_fetch_all_services_by_user,
     dao_fetch_all_services_created_by_user,
     dao_fetch_service_by_id,
-    dao_update_service,
+    dao_update_service
-    delete_service_and_all_associated_db_objects,
 )
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.users_dao import (
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes, get_user_by_email
-    delete_model_user,
-    delete_user_verify_codes,
-    get_user_by_email,
-)
 from app.models import (
-    KEY_TYPE_TEST,
+    PROVIDERS,
     NOTIFICATION_CREATED,
+    KEY_TYPE_TEST,
     SMS_TYPE,
-    AnnualBilling,
+    EMAIL_TYPE,
-    Domain,
+    LETTER_TYPE,
-    EmailBranding,
+    User,
-    LetterBranding,
     Notification,
     Organisation,
-    Permission,
+    Domain,
     Service,
-    User,
+    EmailBranding,
+    LetterBranding,
 )
-from app.utils import get_london_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before
 
 
 @click.group(name='command', help='Additional commands')
@@ -81,28 +73,32 @@ class notify_command:
         self.name = name
 
     def __call__(self, func):
-        decorators = [
+        # we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
-            functools.wraps(func),  # carry through function name, docstrings, etc.
+        # we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
-            click.command(name=self.name),  # turn it into a click.Command
+        # Then we need to turn it into a click.Command - that's what command_group.add_command expects.
-        ]
+        @click.command(name=self.name)
-
+        @functools.wraps(func)
-        # in the test environment the app context is already provided and having
+        @flask.cli.with_appcontext
-        # another will lead to the test db connection being closed prematurely
-        if os.getenv('NOTIFY_ENVIRONMENT', '') != 'test':
-            # with_appcontext ensures the config is loaded, db connected, etc.
-            decorators.insert(0, flask.cli.with_appcontext)
-
         def wrapper(*args, **kwargs):
             return func(*args, **kwargs)
 
-        for decorator in decorators:
-            # this syntax is equivalent to e.g. "@flask.cli.with_appcontext"
-            wrapper = decorator(wrapper)
-
         command_group.add_command(wrapper)
+
         return wrapper
 
 
+@notify_command()
+@click.option('-p', '--provider_name', required=True, type=click.Choice(PROVIDERS))
+@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals', type=float)
+@click.option('-d', '--valid_from', required=True, type=click_dt(format='%Y-%m-%dT%H:%M:%S'))
+def create_provider_rates(provider_name, cost, valid_from):
+    """
+    Backfill rates for a given provider
+    """
+    cost = Decimal(cost)
+    dao_create_provider_rates(provider_name, valid_from, cost)
+
+
 @notify_command()
 @click.option('-u', '--user_email_prefix', required=True, help="""
     Functional test user email prefix. eg "notify-test-preview"
@@ -223,43 +219,255 @@ def fix_notification_statuses_not_in_sync():
         result = db.session.execute(subq_hist).fetchall()
 
 
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+
+    WARNING: This does not overwrite existing data. You need to delete
+             the existing data or Performance Platform will double-count.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_processing_time(start_date, end_date):
+    """
+    Send historical processing time to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+        # because the tz conversion funcs talk about midnight, and the midnight before last,
+        # we want to pretend we're running this from the next morning, so add one.
+        process_date = start_date + timedelta(days=i + 1)
+
+        process_start_date = get_midnight_for_day_before(process_date)
+        process_end_date = get_london_midnight_in_utc(process_date)
+
+        print('Sending notification processing-time for {} - {}'.format(
+            process_start_date.isoformat(),
+            process_end_date.isoformat()
+        ))
+        send_processing_time_for_start_and_end(process_start_date, process_end_date)
+
+
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='list-routes')
+def list_routes():
+    """List URLs of all application routes."""
+    for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
+        print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
+
+
 @notify_command(name='insert-inbound-numbers')
 @click.option('-f', '--file_name', required=True,
               help="""Full path of the file to upload, file is a contains inbound numbers,
               one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
+    file = open(file_name)
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
 
-        for line in file:
+    for line in file:
-            line = line.strip()
+        print(line)
-            if line:
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
-                print(line)
+        db.session.commit()
-                db.session.execute(sql.format(uuid.uuid4(), line))
+    file.close()
-                db.session.commit()
 
 
-@notify_command(name='replay-create-pdf-for-templated-letter')
+@notify_command(name='replay-create-pdf-letters')
 @click.option('-n', '--notification_id', type=click.UUID, required=True,
               help="Notification id of the letter that needs the get_pdf_for_templated_letter task replayed")
-def replay_create_pdf_for_templated_letter(notification_id):
+def replay_create_pdf_letters(notification_id):
     print("Create task to get_pdf_for_templated_letter for notification: {}".format(notification_id))
     get_pdf_for_templated_letter.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)
 
 
-@notify_command(name='recreate-pdf-for-precompiled-or-uploaded-letter')
+@notify_command(name='replay-service-callbacks')
-@click.option('-n', '--notification_id', type=click.UUID, required=True,
+@click.option('-f', '--file_name', required=True,
-              help="Notification ID of the precompiled or uploaded letter")
+              help="""Full path of the file to upload, file is a contains client references of
-def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id):
+              notifications that need the status to be sent to the service.""")
-    print(f"Call resanitise_pdf task for notification: {notification_id}")
+@click.option('-s', '--service_id', required=True,
-    resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS)
+              help="""The service that the callbacks are for""")
+def replay_service_callbacks(file_name, service_id):
+    print("Start send service callbacks for service: ", service_id)
+    callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
+    if not callback_api:
+        print("Callback api was not found for service: {}".format(service_id))
+        return
+
+    errors = []
+    notifications = []
+    file = open(file_name)
+
+    for ref in file:
+        try:
+            notification = Notification.query.filter_by(client_reference=ref.strip()).one()
+            notifications.append(notification)
+        except NoResultFound:
+            errors.append("Reference: {} was not found in notifications.".format(ref))
+
+    for e in errors:
+        print(e)
+    if errors:
+        raise Exception("Some notifications for the given references were not found")
+
+    for n in notifications:
+        data = {
+            "notification_id": str(n.id),
+            "notification_client_reference": n.client_reference,
+            "notification_to": n.to,
+            "notification_status": n.status,
+            "notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
+            "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
+            "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
+            "notification_type": n.notification_type,
+            "service_callback_api_url": callback_api.url,
+            "service_callback_api_bearer_token": callback_api.bearer_token,
+        }
+        encrypted_status_update = encryption.encrypt(data)
+        send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
+                                                    queue=QueueNames.CALLBACKS)
+
+    print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
+        service_id, len(notifications)))
 
 
 def setup_commands(application):
     application.cli.add_command(command_group)
 
 
+@notify_command(name='migrate-data-to-ft-billing')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@statsd(namespace="tasks")
+def migrate_data_to_ft_billing(start_date, end_date):
+
+    current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
+
+    process_date = start_date
+    total_updated = 0
+
+    while process_date < end_date:
+        start_time = datetime.utcnow()
+        # migrate data into ft_billing, upserting the data if it the record already exists
+        sql = \
+            """
+            insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
+                international, billable_units, notifications_sent, rate, postage, created_at)
+                select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
+                    case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
+                from (
+                    select
+                        n.id,
+                        (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
+                        coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
+                        coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
+                        n.notification_type,
+                        coalesce(n.sent_by, (
+                        case
+                        when notification_type = 'sms' then
+                            coalesce(sent_by, 'unknown')
+                        when notification_type = 'letter' then
+                            coalesce(sent_by, 'dvla')
+                        else
+                            coalesce(sent_by, 'ses')
+                        end )) as provider,
+                        coalesce(n.rate_multiplier,1) as rate_multiplier,
+                        s.crown,
+                        coalesce((select rates.rate from rates
+                        where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
+                        order by rates.valid_from desc limit 1), 0) as sms_rate,
+                        coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
+                        and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
+                        and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
+                        as letter_rate,
+                        coalesce(n.international, false) as international,
+                        n.billable_units,
+                        1 as notifications_sent,
+                        coalesce(n.postage, 'none') as postage,
+                        now() as created_at
+                    from public.notification_history n
+                    left join services s on s.id = n.service_id
+                    where n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
+                        and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
+                        at time zone 'UTC'
+                        and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
+                    ) as individual_record
+                group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sms_rate, letter_rate, postage, created_at
+                order by bst_date
+            on conflict on constraint ft_billing_pkey do update set
+             billable_units = excluded.billable_units,
+             notifications_sent = excluded.notifications_sent,
+             rate = excluded.rate,
+             updated_at = now()
+            """
+
+        result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
+        db.session.commit()
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
+
+        process_date += timedelta(days=1)
+
+        total_updated += result.rowcount
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
 @notify_command(name='rebuild-ft-billing-for-day')
 @click.option('-s', '--service_id', required=False, type=click.UUID)
 @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -299,6 +507,29 @@ def rebuild_ft_billing_for_day(service_id, day):
             rebuild_ft_data(day, row.service_id)
 
 
+@notify_command(name='migrate-data-to-ft-notification-status')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
+@statsd(namespace="tasks")
+def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
+    notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
+
+    start_date = start_date.date()
+    end_date = end_date.date()
+    for day_diff in range((end_date - start_date).days + 1):
+        process_day = start_date + timedelta(days=day_diff)
+        for notification_type in notification_types:
+            print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
+                process_day,
+                notification_type
+            ))
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
+            )
+
+
 @notify_command(name='bulk-invite-user-to-service')
 @click.option('-f', '--file_name', required=True,
               help="Full path of the file containing a list of email address for people to invite to a service")
@@ -315,7 +546,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi
     #  platform_admin
     #  view_activity
     # "send_texts,send_emails,send_letters,view_activity"
-    from app.service_invite.rest import create_invited_user
+    from app.invite.rest import create_invited_user
     file = open(file_name)
     for email_address in file:
         data = {
@@ -522,39 +753,6 @@ def populate_organisations_from_file(file_name):
                         db.session.rollback()
 
 
-@notify_command(name='populate-organisation-agreement-details-from-file')
-@click.option('-f', '--file_name', required=True,
-              help="CSV file containing id, agreement_signed_version, "
-              "agreement_signed_on_behalf_of_name, agreement_signed_at")
-def populate_organisation_agreement_details_from_file(file_name):
-    """
-    The input file should be a comma separated CSV file with a header row and 4 columns
-    id: the organisation ID
-    agreement_signed_version
-    agreement_signed_on_behalf_of_name
-    agreement_signed_at: The date the agreement was signed in the format of 'dd/mm/yyyy'
-    """
-    with open(file_name) as f:
-        csv_reader = csv.reader(f)
-
-        # ignore the header row
-        next(csv_reader)
-
-        for row in csv_reader:
-            org = dao_get_organisation_by_id(row[0])
-
-            current_app.logger.info(f"Updating {org.name}")
-
-            assert org.agreement_signed
-
-            org.agreement_signed_version = float(row[1])
-            org.agreement_signed_on_behalf_of_name = row[2].strip()
-            org.agreement_signed_at = datetime.strptime(row[3], "%d/%m/%Y")
-
-            db.session.add(org)
-            db.session.commit()
-
-
 @notify_command(name='get-letter-details-from-zips-sent-file')
 @click.argument('file_paths', required=True, nargs=-1)
 @statsd(namespace="tasks")
@@ -575,24 +773,9 @@ def get_letter_details_from_zips_sent_file(file_paths):
         rows_from_file.extend(json.loads(file_contents))
 
     notification_references = tuple(row[18:34] for row in rows_from_file)
-    get_letters_data_from_references(notification_references)
 
-
-@notify_command(name='get-notification-and-service-ids-for-letters-that-failed-to-print')
-@click.option('-f', '--file_name', required=True,
-              help="""Full path of the file to upload, file should contain letter filenames, one per line""")
-def get_notification_and_service_ids_for_letters_that_failed_to_print(file_name):
-    print("Getting service and notification ids for letter filenames list {}".format(file_name))
-    file = open(file_name)
-    references = tuple([row[7:23] for row in file])
-
-    get_letters_data_from_references(tuple(references))
-    file.close()
-
-
-def get_letters_data_from_references(notification_references):
     sql = """
-        SELECT id, service_id, template_id, reference, job_id, created_at
+        SELECT id, service_id, reference, job_id, created_at
         FROM notifications
         WHERE reference IN :notification_references
         ORDER BY service_id, job_id"""
@@ -600,7 +783,7 @@ def get_letters_data_from_references(notification_references):
 
     with open('zips_sent_details.csv', 'w') as csvfile:
         csv_writer = csv.writer(csvfile)
-        csv_writer.writerow(['notification_id', 'service_id', 'template_id', 'reference', 'job_id', 'created_at'])
+        csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at'])
 
         for row in result:
             csv_writer.writerow(row)
@@ -729,115 +912,3 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
-
-
-@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing_with_the_previous_years_allowance(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
-@notify_command(name='populate-annual-billing-with-defaults')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2021""")
-@click.option('-m', '--missing-services-only', default=True, type=bool,
-              help="""If true then only populate services missing from annual billing for the year.
-                      If false populate the default values for all active services.""")
-def populate_annual_billing_with_defaults(year, missing_services_only):
-    """
-    Add or update annual billing with free allowance defaults for all active services.
-    The default free allowance limits are in: app/dao/annual_billing_dao.py:57.
-
-    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
-    This is useful to prevent overriding any services that have a free allowance that is not the default.
-
-    If missing_services_only is false then add or update annual billing for all active services.
-    This is useful to ensure all services start the new year with the correct annual billing.
-    """
-    if missing_services_only:
-        active_services = Service.query.filter(
-            Service.active
-        ).outerjoin(
-            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
-        ).filter(
-            AnnualBilling.id == None  # noqa
-        ).all()
-    else:
-        active_services = Service.query.filter(
-            Service.active
-        ).all()
-    previous_year = year - 1
-    services_with_zero_free_allowance = db.session.query(AnnualBilling.service_id).filter(
-        AnnualBilling.financial_year_start == previous_year,
-        AnnualBilling.free_sms_fragment_limit == 0
-    ).all()
-
-    for service in active_services:
-
-        # If a service has free_sms_fragment_limit for the previous year
-        # set the free allowance for this year to 0 as well.
-        # Else use the default free allowance for the service.
-        if service.id in [x.service_id for x in services_with_zero_free_allowance]:
-            print(f'update service {service.id} to 0')
-            dao_create_or_update_annual_billing_for_year(
-                service_id=service.id,
-                free_sms_fragment_limit=0,
-                financial_year_start=year
-            )
-        else:
-            print(f'update service {service.id} with default')
-            set_default_free_allowance_for_service(service, year)
-
-
-@click.option('-u', '--user-id', required=True)
-@notify_command(name='local-dev-broadcast-permissions')
-def local_dev_broadcast_permissions(user_id):
-    if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
-    user = User.query.filter_by(id=user_id).one()
-
-    user_broadcast_services = Service.query.filter(
-        Service.permissions.any(permission='broadcast'),
-        Service.users.any(id=user_id)
-    )
-
-    for service in user_broadcast_services:
-        permission_list = [
-            Permission(service_id=service.id, user_id=user_id, permission=permission)
-            for permission in [
-                'reject_broadcasts', 'cancel_broadcasts',  # required to create / approve
-                'create_broadcasts', 'approve_broadcasts',  # minimum for testing
-                'manage_templates',  # unlikely but might be useful
-                'view_activity',  # normally added on invite / service creation
-            ]
-        ]
-
-        permission_dao.set_user_service_permission(
-            user, service, permission_list, _commit=True, replace=True
-        )

app/complaint/complaint_rest.py

@@ -3,10 +3,7 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request
 
 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import (
+from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
-    fetch_count_of_complaints,
-    fetch_paginated_complaints,
-)
 from app.errors import register_errors
 from app.schema_validation import validate
 from app.utils import pagination_links

app/complaint/complaint_schema.py

@@ -1,6 +1,6 @@
 
 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "complaint count request schema",
     "type": "object",
     "title": "Complaint count request",

app/config.py

@@ -1,6 +1,6 @@
-import json
-import os
 from datetime import timedelta
+import os
+import json
 
 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -27,15 +27,11 @@ class QueueNames(object):
     PROCESS_FTP = 'process-ftp-tasks'
     CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
     CALLBACKS = 'service-callbacks'
-    CALLBACKS_RETRY = 'service-callbacks-retry'
     LETTERS = 'letter-tasks'
     SMS_CALLBACKS = 'sms-callbacks'
     ANTIVIRUS = 'antivirus-tasks'
     SANITISE_LETTERS = 'sanitise-letter-tasks'
     SAVE_API_EMAIL = 'save-api-email-tasks'
-    SAVE_API_SMS = 'save-api-sms-tasks'
-    BROADCASTS = 'broadcast-tasks'
-    GOVUK_ALERTS = 'govuk-alerts'
 
     @staticmethod
     def all_queues():
@@ -52,83 +48,62 @@ class QueueNames(object):
             QueueNames.NOTIFY,
             QueueNames.CREATE_LETTERS_PDF,
             QueueNames.CALLBACKS,
-            QueueNames.CALLBACKS_RETRY,
             QueueNames.LETTERS,
             QueueNames.SMS_CALLBACKS,
-            QueueNames.SAVE_API_EMAIL,
+            QueueNames.SAVE_API_EMAIL
-            QueueNames.SAVE_API_SMS,
-            QueueNames.BROADCASTS,
         ]
 
 
-class BroadcastProvider:
-    EE = 'ee'
-    VODAFONE = 'vodafone'
-    THREE = 'three'
-    O2 = 'o2'
-
-    PROVIDERS = [EE, VODAFONE, THREE, O2]
-
-
 class TaskNames(object):
     PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
     ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
     SCAN_FILE = 'scan-file'
     SANITISE_LETTER = 'sanitise-and-upload-letter'
     CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter'
-    PUBLISH_GOVUK_ALERTS = 'publish-govuk-alerts'
-    RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter'
 
 
 class Config(object):
     # URL of admin app
-    ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL')
+    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
 
     # URL of api app (on AWS this is the internal api endpoint)
-    API_HOST_NAME = os.environ.get('API_HOST_NAME')
+    API_HOST_NAME = os.getenv('API_HOST_NAME')
 
     # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = 'notify-admin'
+    API_INTERNAL_SECRETS = json.loads(os.environ.get('API_INTERNAL_SECRETS', '[]'))
-    GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove?
-
-    INTERNAL_CLIENT_API_KEYS = json.loads(
-        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}')
-    ) # TODO: handled by varsfile?
 
     # encyption secret/salt
-    ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
+    SECRET_KEY = os.getenv('SECRET_KEY')
-    SECRET_KEY = os.environ.get('SECRET_KEY')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
-    DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT')
 
     # DB conection string
-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
-    
-    # AWS SMS
-    AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION")
-    AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER")
 
     # MMG API Key
-    MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder')
+    MMG_API_KEY = os.getenv('MMG_API_KEY')
 
     # Firetext API Key
-    FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
+    FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
-    FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
 
     # Prefix to identify queues in SQS
-    NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX')
+    NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
 
     # URL of redis instance
-    REDIS_URL = os.environ.get('REDIS_URL')
+    REDIS_URL = os.getenv('REDIS_URL')
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = False
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
+
     # Zendesk
     ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')
 
     # Logging
     DEBUG = False
-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH')
+    NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')
 
     # Cronitor
     CRONITOR_ENABLED = False
@@ -142,7 +117,8 @@ class Config(object):
     ###########################
 
     NOTIFY_ENVIRONMENT = 'development'
-    AWS_REGION = 'us-west-2'
+    ADMIN_CLIENT_USER_NAME = 'notify-admin'
+    AWS_REGION = 'eu-west-1'
     INVITATION_EXPIRATION_DAYS = 2
     NOTIFY_APP_NAME = 'api'
     SQLALCHEMY_RECORD_QUERIES = False
@@ -155,10 +131,7 @@ class Config(object):
     API_PAGE_SIZE = 250
     TEST_MESSAGE_FILENAME = 'Test message'
     ONE_OFF_MESSAGE_FILENAME = 'Report'
-    MAX_VERIFY_CODE_COUNT = 5
+    MAX_VERIFY_CODE_COUNT = 10
-    MAX_FAILED_LOGIN_COUNT = 10
-    
-    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
 
     # be careful increasing this size without being sure that we won't see slowness in pysftp
     MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
@@ -175,7 +148,6 @@ class Config(object):
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
     NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
     INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
-    BROADCAST_INVITATION_EMAIL_TEMPLATE_ID = '46152f7c-6901-41d5-8590-a5624d0d4359'
     SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
     EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
     NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
@@ -190,171 +162,152 @@ class Config(object):
     MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
     MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
+    MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
-    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
-    # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
 
-    CELERY = {
+    BROKER_URL = 'sqs://'
-        'broker_url': 'sqs://',
+    BROKER_TRANSPORT_OPTIONS = {
-        'broker_transport_options': {
+        'region': AWS_REGION,
-            'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
-            'visibility_timeout': 310,
+        'visibility_timeout': 310,
-            'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
-        },
-        'timezone': 'Europe/London',
-        'imports': [
-            'app.celery.tasks',
-            'app.celery.scheduled_tasks',
-            'app.celery.reporting_tasks',
-            'app.celery.nightly_tasks',
-        ],
-        # this is overriden by the -Q command, but locally, we should read from all queues
-        'task_queues': [
-            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
-        ],
-        'beat_schedule': {
-            # app/celery/scheduled_tasks.py
-            'run-scheduled-jobs': {
-                'task': 'run-scheduled-jobs',
-                'schedule': crontab(minute='0,15,30,45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-verify-codes': {
-                'task': 'delete-verify-codes',
-                'schedule': timedelta(minutes=63),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-invitations': {
-                'task': 'delete-invitations',
-                'schedule': timedelta(minutes=66),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'switch-current-sms-provider-on-slow-delivery': {
-                'task': 'switch-current-sms-provider-on-slow-delivery',
-                'schedule': crontab(),  # Every minute
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-job-status': {
-                'task': 'check-job-status',
-                'schedule': crontab(),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'tend-providers-back-to-middle': {
-                'task': 'tend-providers-back-to-middle',
-                'schedule': crontab(minute='*/5'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-missing-rows-in-completed-jobs': {
-                'task': 'check-for-missing-rows-in-completed-jobs',
-                'schedule': crontab(minute='*/10'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'replay-created-notifications': {
-                'task': 'replay-created-notifications',
-                'schedule': crontab(minute='0, 15, 30, 45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # app/celery/nightly_tasks.py
-            'timeout-sending-notifications': {
-                'task': 'timeout-sending-notifications',
-                'schedule': crontab(hour=0, minute=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'create-nightly-billing': {
-                'task': 'create-nightly-billing',
-                'schedule': crontab(hour=0, minute=15),
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'create-nightly-notification-status': {
-                'task': 'create-nightly-notification-status',
-                'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-notifications-older-than-retention': {
-                'task': 'delete-notifications-older-than-retention',
-                'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-inbound-sms': {
-                'task': 'delete-inbound-sms',
-                'schedule': crontab(hour=1, minute=40),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'save-daily-notification-processing-time': {
-                'task': 'save-daily-notification-processing-time',
-                'schedule': crontab(hour=2, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove_sms_email_jobs': {
-                'task': 'remove_sms_email_jobs',
-                'schedule': crontab(hour=4, minute=0),
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'remove_letter_jobs': {
-                'task': 'remove_letter_jobs',
-                'schedule': crontab(hour=4, minute=20),
-                # since we mark jobs as archived
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'check-if-letters-still-in-created': {
-                'task': 'check-if-letters-still-in-created',
-                'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-if-letters-still-pending-virus-check': {
-                'task': 'check-if-letters-still-pending-virus-check',
-                'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
-                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
-                'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-letter-notifications-still-sending': {
-                'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=17, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # The collate-letter-pdf does assume it is called in an hour that BST does not make a
-            # difference to the truncate date which translates to the filename to process
-            'collate-letter-pdfs-to-be-sent': {
-                'task': 'collate-letter-pdfs-to-be-sent',
-                'schedule': crontab(hour=17, minute=50),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-no-letter-ack-file': {
-                'task': 'raise-alert-if-no-letter-ack-file',
-                'schedule': crontab(hour=23, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'trigger-link-tests': {
-                'task': 'trigger-link-tests',
-                'schedule': timedelta(minutes=15),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'auto-expire-broadcast-messages': {
-                'task': 'auto-expire-broadcast-messages',
-                'schedule': timedelta(minutes=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove-yesterdays-planned-tests-on-govuk-alerts': {
-                'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
-                'schedule': crontab(hour=00, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-        }
     }
-
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = 'Europe/London'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TASK_SERIALIZER = 'json'
+    # on reporting worker, restart workers after each task is executed to help prevent memory leaks
+    CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD')
     # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
-    if os.environ.get('CELERYD_PREFETCH_MULTIPLIER'):
+    if os.getenv('CELERYD_PREFETCH_MULTIPLIER'):
-        CELERY['worker_prefetch_multiplier'] = os.environ.get('CELERYD_PREFETCH_MULTIPLIER')
+        CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
+    CELERY_IMPORTS = (
+        'app.celery.tasks',
+        'app.celery.scheduled_tasks',
+        'app.celery.reporting_tasks',
+        'app.celery.nightly_tasks',
+    )
+    CELERYBEAT_SCHEDULE = {
+        # app/celery/scheduled_tasks.py
+        'run-scheduled-jobs': {
+            'task': 'run-scheduled-jobs',
+            'schedule': crontab(minute=1),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-verify-codes': {
+            'task': 'delete-verify-codes',
+            'schedule': timedelta(minutes=63),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-invitations': {
+            'task': 'delete-invitations',
+            'schedule': timedelta(minutes=66),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-job-status': {
+            'task': 'check-job-status',
+            'schedule': crontab(),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'tend-providers-back-to-middle': {
+            'task': 'tend-providers-back-to-middle',
+            'schedule': crontab(minute='*/5'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-missing-rows-in-completed-jobs': {
+            'task': 'check-for-missing-rows-in-completed-jobs',
+            'schedule': crontab(minute='*/10'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'replay-created-notifications': {
+            'task': 'replay-created-notifications',
+            'schedule': crontab(minute='0, 15, 30, 45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # app/celery/nightly_tasks.py
+        'timeout-sending-notifications': {
+            'task': 'timeout-sending-notifications',
+            'schedule': crontab(hour=0, minute=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=0, minute=15),
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'create-nightly-notification-status': {
+            'task': 'create-nightly-notification-status',
+            'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'delete-notifications-older-than-retention': {
+            'task': 'delete-notifications-older-than-retention',
+            'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(hour=1, minute=40),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'remove_sms_email_jobs': {
+            'task': 'remove_sms_email_jobs',
+            'schedule': crontab(hour=4, minute=0),
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'remove_letter_jobs': {
+            'task': 'remove_letter_jobs',
+            'schedule': crontab(hour=4, minute=20),
+            # since we mark jobs as archived
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+            'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+            'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-letter-notifications-still-sending': {
+            'task': 'raise-alert-if-letter-notifications-still-sending',
+            'schedule': crontab(hour=15, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-to-be-sent': {
+            'task': 'collate-letter-pdfs-to-be-sent',
+            'schedule': crontab(hour=17, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-no-letter-ack-file': {
+            'task': 'raise-alert-if-no-letter-ack-file',
+            'schedule': crontab(hour=23, minute=00),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+    }
+    CELERY_QUEUES = []
 
     FROM_NUMBER = 'development'
 
-    STATSD_HOST = os.environ.get('STATSD_HOST')
+    STATSD_HOST = os.getenv('STATSD_HOST')
     STATSD_PORT = 8125
     STATSD_ENABLED = bool(STATSD_HOST)
 
@@ -374,11 +327,15 @@ class Config(object):
     FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
     MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
     MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
-    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
+    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
-    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
+    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
 
     HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
 
+    # Format is as follows:
+    # {"dataset_1": "token_1", ...}
+    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
+
     TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
     TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
 
@@ -388,17 +345,9 @@ class Config(object):
     # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
     MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
     FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
+    SES_STUB_URL = os.environ.get("SES_STUB_URL")
 
-    AWS_REGION = 'us-west-2'
+    AWS_REGION = 'eu-west-1'
-
-    CBC_PROXY_ENABLED = True
-    CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
-    CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '')
-
-    ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE}
-
-    # as defined in api db migration 0331_add_broadcast_org.py
-    BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
 
 
 ######################
@@ -409,10 +358,8 @@ class Development(Config):
     DEBUG = True
     SQLALCHEMY_ECHO = False
 
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
+    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
-
+    CONTACT_LIST_BUCKET_NAME = 'development-contact-list'
-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
     TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
     LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
@@ -421,11 +368,7 @@ class Development(Config):
     TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
 
-    # INTERNAL_CLIENT_API_KEYS = {
+    API_INTERNAL_SECRETS = ['dev-notify-secret-key']
-    #     Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
-    #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
-    # }
-
     SECRET_KEY = 'dev-notify-secret-key'
     DANGEROUS_SALT = 'dev-notify-salt'
 
@@ -434,21 +377,21 @@ class Development(Config):
 
     NOTIFY_ENVIRONMENT = 'development'
     NOTIFY_LOG_PATH = 'application.log'
-    NOTIFY_EMAIL_DOMAIN = "dispostable.com"
+    NOTIFICATION_QUEUE_PREFIX = 'development'
+    NOTIFY_EMAIL_DOMAIN = "notify.tools"
 
-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
+    SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
-    REDIS_URL = os.environ.get('REDIS_URL')
+    REDIS_URL = 'redis://localhost:6379/0'
 
-    ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
+    ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 
-    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
+    for queue in QueueNames.all_queues():
-
+        Config.CELERY_QUEUES.append(
-    API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011')
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
 
+    API_HOST_NAME = "http://localhost:6011"
     API_RATE_LIMIT_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
-
-    CBC_PROXY_ENABLED = False
 
 
 class Test(Development):
@@ -457,12 +400,7 @@ class Test(Development):
     NOTIFY_ENVIRONMENT = 'test'
     TESTING = True
 
-    HIGH_VOLUME_SERVICE = [
+    HIGH_VOLUME_SERVICE = ['941b6f9a-50d7-4742-8d50-f365ca74bf27']
-        '941b6f9a-50d7-4742-8d50-f365ca74bf27',
-        '63f95b86-2d19-4497-b8b2-ccf25457df4e',
-        '7e5950cb-9954-41f5-8376-962b8c8555cf',
-        '10d1b9c9-0072-4fa9-ae1c-595e333841da',
-    ]
 
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
@@ -474,16 +412,18 @@ class Test(Development):
     TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
 
-    # this is overriden in CI
+    # this is overriden in jenkins and on cloudfoundry
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api')
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
 
-    CELERY = {
+    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
-        **Config.CELERY,
-        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
-    }
 
     ANTIVIRUS_ENABLED = True
 
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
     API_RATE_LIMIT_ENABLED = True
     API_HOST_NAME = "http://localhost:6011"
 
@@ -494,9 +434,6 @@ class Test(Development):
     MMG_URL = 'https://example.com/mmg'
     FIRETEXT_URL = 'https://example.com/firetext'
 
-    CBC_PROXY_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
-
 
 class Preview(Config):
     NOTIFY_EMAIL_DOMAIN = 'notify.works'
@@ -530,34 +467,31 @@ class Staging(Config):
     FROM_NUMBER = 'stage'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
+    REDIS_ENABLED = True
+    # SES_STUB_URL = 'https://notify-email-provider-stub-staging.cloudapps.digital/ses'
+    MMG_URL = 'https://notify-sms-provider-stub-staging.cloudapps.digital/mmg'
+    FIRETEXT_URL = 'https://notify-sms-provider-stub-staging.cloudapps.digital/firetext'
 
 
 class Live(Config):
-    NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN')
+    NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
     NOTIFY_ENVIRONMENT = 'live'
-    # buckets
+    CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
-    CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox
+    CONTACT_LIST_BUCKET_NAME = 'production-contact-list'
-    CONTACT_LIST_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
-    # TODO: verify below buckets only used for letters
+    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
-    TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox
+    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
-    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox
+    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
-    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox
+    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
-    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
-    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
+    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
-    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
+    FROM_NUMBER = 'GOVUK'
-    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
+    PERFORMANCE_PLATFORM_ENABLED = True
-    
-    FROM_NUMBER = 'US Notify'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
     SES_STUB_URL = None
-    CRONITOR_ENABLED = True
-    
-    # DEBUG = True
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
 
-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log')
+    CRONITOR_ENABLED = True
-    REDIS_URL = os.environ.get('REDIS_URL')
 
 
 class CloudFoundryConfig(Config):
@@ -577,6 +511,7 @@ class Sandbox(CloudFoundryConfig):
     LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
     INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
     FROM_NUMBER = 'sandbox'
+    REDIS_ENABLED = False
 
 
 configs = {

app/cronitor.py

@@ -1,19 +1,15 @@
-from functools import wraps
-
 import requests
+from functools import wraps
 from flask import current_app
 
 
 def cronitor(task_name):
+    # check if task_name is in config
     def decorator(func):
         def ping_cronitor(command):
             if not current_app.config['CRONITOR_ENABLED']:
                 return
 
-            # it's useful to have a log that a periodic task has started in case it
-            # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
-
             task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
             if not task_slug:
                 current_app.logger.error(
@@ -42,11 +38,13 @@ def cronitor(task_name):
         @wraps(func)
         def inner_decorator(*args, **kwargs):
             ping_cronitor('run')
-            status = 'fail'
             try:
                 ret = func(*args, **kwargs)
                 status = 'complete'
                 return ret
+            except Exception:
+                status = 'fail'
+                raise
             finally:
                 ping_cronitor(status)
 

app/dao/__init__.py

@@ -1,5 +1,4 @@
 from sqlalchemy.exc import SQLAlchemyError
-
 from app import db
 
 

app/dao/annual_billing_dao.py

@@ -1,12 +1,10 @@
-from flask import current_app
-
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
-from app.dao.date_util import get_current_financial_year_start_year
 from app.models import AnnualBilling
+from app.dao.date_util import get_current_financial_year_start_year
 
 
-@autocommit
+@transactional
 def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
     result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
@@ -25,7 +23,7 @@ def dao_get_annual_billing(service_id):
     ).order_by(AnnualBilling.financial_year_start).all()
 
 
-@autocommit
+@transactional
 def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
     AnnualBilling.query.filter(
         AnnualBilling.service_id == service_id,
@@ -51,67 +49,3 @@ def dao_get_all_free_sms_fragment_limit(service_id):
     return AnnualBilling.query.filter_by(
         service_id=service_id,
     ).order_by(AnnualBilling.financial_year_start).all()
-
-
-def set_default_free_allowance_for_service(service, year_start=None):
-    default_free_sms_fragment_limits = {
-        'central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'nhs_local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_gp': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'emergency_service': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'school_or_college': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'other': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-    }
-    if not year_start:
-        year_start = get_current_financial_year_start_year()
-    # handle cases where the year is less than 2020 or greater than 2021
-    if year_start < 2020:
-        year_start = 2020
-    if year_start > 2022:
-        year_start = 2022
-    if service.organisation_type:
-        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
-    else:
-        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
-                                f"{default_free_sms_fragment_limits['other'][year_start]}")
-        free_allowance = default_free_sms_fragment_limits['other'][year_start]
-
-    return dao_create_or_update_annual_billing_for_year(
-        service.id,
-        free_allowance,
-        year_start
-    )

app/dao/api_key_dao.py

@@ -1,14 +1,18 @@
 import uuid
 from datetime import datetime, timedelta
 
-from sqlalchemy import func, or_
-
 from app import db
-from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey
 
+from app.dao.dao_utils import (
+    transactional,
+    version_class
+)
 
-@autocommit
+from sqlalchemy import or_, func
+
+
+@transactional
 @version_class(ApiKey)
 def save_model_api_key(api_key):
     if not api_key.id:
@@ -17,7 +21,7 @@ def save_model_api_key(api_key):
     db.session.add(api_key)
 
 
-@autocommit
+@transactional
 @version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
     api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()

app/dao/broadcast_message_dao.py

@@ -1,105 +0,0 @@
-import uuid
-from datetime import datetime
-
-from sqlalchemy import desc
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    BroadcastEvent,
-    BroadcastMessage,
-    BroadcastProvider,
-    BroadcastProviderMessage,
-    BroadcastProviderMessageNumber,
-    BroadcastProviderMessageStatus,
-    BroadcastStatusType,
-    ServiceBroadcastSettings,
-)
-
-
-def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.id == broadcast_message_id,
-        BroadcastMessage.service_id == service_id
-    ).one()
-
-
-def dao_get_broadcast_message_by_references_and_service_id(references_to_original_broadcast, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.status.in_((
-            BroadcastStatusType.PENDING_APPROVAL,
-            BroadcastStatusType.BROADCASTING,
-        )),
-        BroadcastMessage.reference.in_(references_to_original_broadcast),
-        BroadcastMessage.service_id == service_id
-    ).one()
-
-
-def dao_get_broadcast_event_by_id(broadcast_event_id):
-    return BroadcastEvent.query.filter(BroadcastEvent.id == broadcast_event_id).one()
-
-
-def dao_get_broadcast_messages_for_service(service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.service_id == service_id
-    ).order_by(BroadcastMessage.created_at)
-
-
-def dao_get_all_broadcast_messages():
-    return db.session.query(
-        BroadcastMessage.id,
-        BroadcastMessage.reference,
-        ServiceBroadcastSettings.channel,
-        BroadcastMessage.content,
-        BroadcastMessage.areas,
-        BroadcastMessage.status,
-        BroadcastMessage.starts_at,
-        BroadcastMessage.finishes_at,
-        BroadcastMessage.approved_at,
-        BroadcastMessage.cancelled_at,
-    ).join(
-        ServiceBroadcastSettings, ServiceBroadcastSettings.service_id == BroadcastMessage.service_id
-    ).filter(
-        BroadcastMessage.starts_at >= datetime(2021, 5, 25, 0, 0, 0),
-        BroadcastMessage.stubbed == False,  # noqa
-        BroadcastMessage.status.in_(BroadcastStatusType.LIVE_STATUSES)
-    ).order_by(desc(BroadcastMessage.starts_at)).all()
-
-
-def get_earlier_events_for_broadcast_event(broadcast_event_id):
-    """
-    This is used to build up the references list.
-    """
-    this_event = BroadcastEvent.query.get(broadcast_event_id)
-
-    return BroadcastEvent.query.filter(
-        BroadcastEvent.broadcast_message_id == this_event.broadcast_message_id,
-        BroadcastEvent.sent_at < this_event.sent_at
-    ).order_by(
-        BroadcastEvent.sent_at.asc()
-    ).all()
-
-
-@autocommit
-def create_broadcast_provider_message(broadcast_event, provider):
-    broadcast_provider_message_id = uuid.uuid4()
-    provider_message = BroadcastProviderMessage(
-        id=broadcast_provider_message_id,
-        broadcast_event=broadcast_event,
-        provider=provider,
-        status=BroadcastProviderMessageStatus.SENDING,
-    )
-    db.session.add(provider_message)
-    db.session.commit()
-    provider_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        provider_message_number = BroadcastProviderMessageNumber(
-            broadcast_provider_message_id=broadcast_provider_message_id)
-        db.session.add(provider_message_number)
-        db.session.commit()
-    return provider_message
-
-
-@autocommit
-def update_broadcast_provider_message_status(broadcast_provider_message, *, status):
-    broadcast_provider_message.status = status

app/dao/broadcast_service_dao.py

@@ -1,100 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    BROADCAST_TYPE,
-    EMAIL_AUTH_TYPE,
-    INVITE_PENDING,
-    VIEW_ACTIVITY,
-    ApiKey,
-    InvitedUser,
-    Organisation,
-    Permission,
-    Service,
-    ServiceBroadcastSettings,
-    ServicePermission,
-)
-
-
-@autocommit
-@version_class(Service)
-def set_broadcast_service_type(service, service_mode, broadcast_channel, provider_restriction):
-    insert_or_update_service_broadcast_settings(
-        service, channel=broadcast_channel, provider_restriction=provider_restriction
-    )
-
-    # Remove all permissions and add broadcast permission
-    if not service.has_permission(BROADCAST_TYPE):
-        service_permission = ServicePermission(service_id=service.id, permission=BROADCAST_TYPE)
-        db.session.add(service_permission)
-
-    ServicePermission.query.filter(
-        ServicePermission.service_id == service.id,
-        ServicePermission.permission != BROADCAST_TYPE,
-        # Email auth is an exception to the other service permissions (which relate to what type
-        # of notifications a service can send) where a broadcast service is allowed to have the
-        # email auth permission (but doesn't have to)
-        ServicePermission.permission != EMAIL_AUTH_TYPE
-    ).delete()
-
-    # Refresh the service object as it has references to the service permissions but we don't yet
-    # want to commit the permission changes incase all of this needs to rollback
-    db.session.refresh(service)
-
-    # Set service count as live false always
-    service.count_as_live = False
-
-    # Set service into training mode or live mode
-    if service_mode == "live":
-        if service.restricted:
-            # Only update the go live at timestamp if this if moving from training mode
-            # to live mode, not if it's moving from one type of live mode service to another
-            service.go_live_at = datetime.utcnow()
-        service.restricted = False
-    else:
-        service.restricted = True
-        service.go_live_at = None
-
-    # Remove all user permissions apart from view_activity for the service users and invited users
-    Permission.query.filter(
-        Permission.service_id == service.id,
-        Permission.permission != VIEW_ACTIVITY
-    ).delete()
-    InvitedUser.query.filter_by(
-        service_id=service.id,
-        status=INVITE_PENDING
-    ).update({'permissions': VIEW_ACTIVITY})
-
-    # Revoke any API keys to avoid a regular API key being used to send alerts
-    ApiKey.query.filter_by(
-        service_id=service.id,
-        expiry_date=None,
-    ).update({
-        ApiKey.expiry_date: datetime.utcnow()
-    })
-
-    # Add service to organisation
-    organisation = Organisation.query.filter_by(
-        id=current_app.config['BROADCAST_ORGANISATION_ID']
-    ).one()
-    service.organisation_id = organisation.id
-    service.organisation_type = organisation.organisation_type
-    service.crown = organisation.crown
-
-    db.session.add(service)
-
-
-def insert_or_update_service_broadcast_settings(service, channel, provider_restriction="all"):
-    if not service.service_broadcast_settings:
-        settings = ServiceBroadcastSettings()
-        settings.service = service
-        settings.channel = channel
-        settings.provider = provider_restriction
-        db.session.add(settings)
-    else:
-        service.service_broadcast_settings.channel = channel
-        service.service_broadcast_settings.provider = provider_restriction
-        db.session.add(service.service_broadcast_settings)

app/dao/complaint_dao.py

@@ -4,12 +4,12 @@ from flask import current_app
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import Complaint
 from app.utils import get_london_midnight_in_utc
 
 
-@autocommit
+@transactional
 def save_complaint(complaint):
     db.session.add(complaint)
 

app/dao/daily_sorted_letter_dao.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import DailySortedLetter
 
 
@@ -13,7 +13,7 @@ def dao_get_daily_sorted_letter_by_billing_day(billing_day):
     ).first()
 
 
-@autocommit
+@transactional
 def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
     '''
     This uses the Postgres upsert to avoid race conditions when two threads try and insert

app/dao/dao_utils.py

@@ -1,20 +1,16 @@
 import itertools
-from contextlib import contextmanager
 from functools import wraps
 
 from app import db
 from app.history_meta import create_history
 
 
-def autocommit(func):
+def transactional(func):
     @wraps(func)
     def commit_or_rollback(*args, **kwargs):
         try:
             res = func(*args, **kwargs)
-
+            db.session.commit()
-            if not db.session.registry().transaction.nested:
-                db.session.commit()
-
             return res
         except Exception:
             db.session.rollback()
@@ -22,20 +18,6 @@ def autocommit(func):
     return commit_or_rollback
 
 
-@contextmanager
-def transaction():
-    try:
-        db.session.begin_nested()
-        yield
-        db.session.commit()
-
-        if not db.session.registry().transaction.nested:
-            db.session.commit()
-    except Exception:
-        db.session.rollback()
-        raise
-
-
 class VersionOptions():
 
     def __init__(self, model_class, history_class=None, must_write_history=True):
@@ -91,9 +73,3 @@ def version_class(*version_options):
 
 def dao_rollback():
     db.session.rollback()
-
-
-@autocommit
-def dao_save_object(obj):
-    # add/update object in db
-    db.session.add(obj)

app/dao/date_util.py

@@ -1,7 +1,7 @@
-from datetime import date, datetime, time, timedelta
+from datetime import datetime, timedelta, date, time
 
+from notifications_utils.timezones import convert_bst_to_utc
 import pytz
-from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 
 
 def get_months_for_financial_year(year):
@@ -22,15 +22,6 @@ def get_financial_year(year):
     return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
 
 
-def get_financial_year_dates(year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    return (
-        convert_utc_to_bst(year_start_datetime).date(),
-        convert_utc_to_bst(year_end_datetime).date()
-    )
-
-
 def get_current_financial_year():
     now = datetime.utcnow()
     current_month = int(now.strftime('%-m'))

app/dao/email_branding_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import EmailBranding
 
 
@@ -15,12 +15,12 @@ def dao_get_email_branding_by_name(email_branding_name):
     return EmailBranding.query.filter_by(name=email_branding_name).first()
 
 
-@autocommit
+@transactional
 def dao_create_email_branding(email_branding):
     db.session.add(email_branding)
 
 
-@autocommit
+@transactional
 def dao_update_email_branding(email_branding, **kwargs):
     for key, value in kwargs.items():
         setattr(email_branding, key, value or None)

app/dao/fact_notification_status_dao.py

@@ -1,91 +1,115 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time
 
-from sqlalchemy import Date, case, func
+from flask import current_app
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.timezones import convert_bst_to_utc
+from sqlalchemy import case, func, Date
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import extract, literal
+from sqlalchemy.sql.expression import literal, extract
 from sqlalchemy.types import DateTime, Integer
 
 from app import db
-from app.dao.dao_utils import autocommit
 from app.models import (
-    KEY_TYPE_NORMAL,
+    FactNotificationStatus,
-    KEY_TYPE_TEAM,
     KEY_TYPE_TEST,
+    Notification,
     NOTIFICATION_CANCELLED,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_FAILED,
     NOTIFICATION_PENDING,
-    NOTIFICATION_PERMANENT_FAILURE,
     NOTIFICATION_SENDING,
     NOTIFICATION_SENT,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    FactNotificationStatus,
+    NOTIFICATION_PERMANENT_FAILURE,
-    Notification,
-    NotificationAllTimeView,
     Service,
     Template,
 )
+from app.dao.dao_utils import transactional
 from app.utils import (
     get_london_midnight_in_utc,
-    get_london_month_from_utc_column,
     midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
 )
 
 
-@autocommit
+@statsd(namespace="dao")
-def update_fact_notification_status(process_day, notification_type, service_id):
+def fetch_notification_status_for_day(process_day, notification_type):
-    start_date = get_london_midnight_in_utc(process_day)
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
-    end_date = get_london_midnight_in_utc(process_day + timedelta(days=1))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
 
-    # delete any existing rows in case some no longer exist e.g. if all messages are sent
+    current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
-    FactNotificationStatus.query.filter(
-        FactNotificationStatus.bst_date == process_day,
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.service_id == service_id,
-    ).delete()
 
+    all_data_for_process_day = []
+    services = Service.query.all()
+    # for each service query notifications or notification_history for the day, depending on their data retention
+    for service in services:
+        table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
+
+        data_for_service_and_type = query_for_fact_status_data(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+            notification_type=notification_type,
+            service_id=service.id
+        )
+
+        all_data_for_process_day += data_for_service_and_type
+
+    return all_data_for_process_day
+
+
+@statsd(namespace="dao")
+def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
     query = db.session.query(
-        literal(process_day).label("process_day"),
+        table.template_id,
-        NotificationAllTimeView.template_id,
+        table.service_id,
-        literal(service_id).label("service_id"),
+        func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
-        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        table.key_type,
-        literal(notification_type).label("notification_type"),
+        table.status,
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status,
         func.count().label('notification_count')
     ).filter(
-        NotificationAllTimeView.created_at >= start_date,
+        table.created_at >= start_date,
-        NotificationAllTimeView.created_at < end_date,
+        table.created_at < end_date,
-        NotificationAllTimeView.notification_type == notification_type,
+        table.notification_type == notification_type,
-        NotificationAllTimeView.service_id == service_id,
+        table.service_id == service_id,
-        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        table.key_type != KEY_TYPE_TEST
     ).group_by(
-        NotificationAllTimeView.template_id,
+        table.template_id,
-        NotificationAllTimeView.template_id,
+        table.service_id,
         'job_id',
-        NotificationAllTimeView.key_type,
+        table.key_type,
-        NotificationAllTimeView.status
+        table.status
     )
+    return query.all()
 
-    db.session.connection().execute(
+
-        insert(FactNotificationStatus.__table__).from_select(
+@statsd(namespace="dao")
-            [
+@transactional
-                FactNotificationStatus.bst_date,
+def update_fact_notification_status(data, process_day, notification_type):
-                FactNotificationStatus.template_id,
+    table = FactNotificationStatus.__table__
-                FactNotificationStatus.service_id,
+    FactNotificationStatus.query.filter(
-                FactNotificationStatus.job_id,
+        FactNotificationStatus.bst_date == process_day,
-                FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_type == notification_type
-                FactNotificationStatus.key_type,
+    ).delete()
-                FactNotificationStatus.notification_status,
+
-                FactNotificationStatus.notification_count
+    for row in data:
-            ],
+        stmt = insert(table).values(
-            query
+            bst_date=process_day,
+            template_id=row.template_id,
+            service_id=row.service_id,
+            job_id=row.job_id,
+            notification_type=notification_type,
+            key_type=row.key_type,
+            notification_status=row.status,
+            notification_count=row.notification_count,
         )
-    )
+        db.session.connection().execute(stmt)
 
 
+@statsd(namespace="dao")
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
     return db.session.query(
         func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
@@ -104,6 +128,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
     ).all()
 
 
+@statsd(namespace="dao")
 def fetch_notification_status_for_service_for_day(bst_day, service_id):
     return db.session.query(
         # return current month as a datetime so the data has the same shape as the ft_notification_status query
@@ -122,6 +147,7 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
     ).all()
 
 
+@statsd(namespace="dao")
 def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
     start_date = midnight_n_days_ago(limit_days)
     now = datetime.utcnow()
@@ -174,6 +200,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
     ).all()
 
 
+@statsd(namespace="dao")
 def fetch_notification_status_totals_for_all_services(start_date, end_date):
     stats = db.session.query(
         FactNotificationStatus.notification_type.label('notification_type'),
@@ -222,6 +249,7 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
     return query.all()
 
 
+@statsd(namespace="dao")
 def fetch_notification_statuses_for_job(job_id):
     return db.session.query(
         FactNotificationStatus.notification_status.label('status'),
@@ -233,6 +261,7 @@ def fetch_notification_statuses_for_job(job_id):
     ).all()
 
 
+@statsd(namespace="dao")
 def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True):
     stats = db.session.query(
         FactNotificationStatus.service_id.label('service_id'),
@@ -327,6 +356,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
     return query.all()
 
 
+@statsd(namespace="dao")
 def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service
     stats = db.session.query(
@@ -411,39 +441,20 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     return query.all()
 
 
-def get_total_notifications_for_date_range(start_date, end_date):
+@statsd(namespace="dao")
-    query = db.session.query(
+def get_total_sent_notifications_for_day_and_type(day, notification_type):
-        FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),
+    result = db.session.query(
-        func.sum(case(
+        func.sum(FactNotificationStatus.notification_count).label('count')
-            [
-                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('emails'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('sms'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('letters'),
     ).filter(
+        FactNotificationStatus.notification_type == notification_type,
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
-    ).group_by(
+        FactNotificationStatus.bst_date == day,
-        FactNotificationStatus.bst_date
+    ).scalar()
-    ).order_by(
+
-        FactNotificationStatus.bst_date
+    return result or 0
-    )
-    if start_date and end_date:
-        query = query.filter(
-            FactNotificationStatus.bst_date >= start_date,
-            FactNotificationStatus.bst_date <= end_date
-        )
-    return query.all()
 
 
+@statsd(namespace="dao")
 def fetch_monthly_notification_statuses_per_service(start_date, end_date):
     return db.session.query(
         func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'),

app/dao/fact_processing_time_dao.py

@@ -1,53 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import case
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import FactProcessingTime
-
-
-@autocommit
-def insert_update_processing_time(processing_time):
-    '''
-    This uses the Postgres upsert to avoid race conditions when two threads try and insert
-    at the same row. The excluded object refers to values that we tried to insert but were
-    rejected.
-    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    '''
-    table = FactProcessingTime.__table__
-    stmt = insert(table).values(
-        bst_date=processing_time.bst_date,
-        messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs
-    )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.bst_date],
-        set_={
-            'messages_total': stmt.excluded.messages_total,
-            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
-            'updated_at': datetime.utcnow()
-        }
-    )
-    db.session.connection().execute(stmt)
-
-
-def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactProcessingTime.bst_date.cast(db.Text).label("date"),
-        FactProcessingTime.messages_total,
-        FactProcessingTime.messages_within_10_secs,
-        case([
-            (
-                FactProcessingTime.messages_total > 0,
-                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
-            ),
-            (FactProcessingTime.messages_total == 0, 100.0)
-        ]).label("percentage")
-    ).filter(
-        FactProcessingTime.bst_date >= start_date,
-        FactProcessingTime.bst_date <= end_date
-    ).order_by(FactProcessingTime.bst_date)
-
-    return query.all()

app/dao/inbound_numbers_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import InboundNumber
 
 
@@ -19,13 +19,13 @@ def dao_get_inbound_number(inbound_number_id):
     return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_to_service(service_id, inbound_number):
     inbound_number.service_id = service_id
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_active_flag(service_id, active):
     inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
     inbound_number.active = active
@@ -33,7 +33,7 @@ def dao_set_inbound_number_active_flag(service_id, active):
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_allocate_number_for_service(service_id, inbound_number_id):
     updated = InboundNumber.query.filter_by(
         id=inbound_number_id,

app/dao/inbound_sms_dao.py

@@ -1,21 +1,16 @@
 from flask import current_app
-from sqlalchemy import and_, desc
+from notifications_utils.statsd_decorators import statsd
-from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import desc, and_
 from sqlalchemy.orm import aliased
+from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
-from app.models import (
+from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
-    SMS_TYPE,
-    InboundSms,
-    InboundSmsHistory,
-    Service,
-    ServiceDataRetention,
-)
 from app.utils import midnight_n_days_ago
 
 
-@autocommit
+@transactional
 def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
@@ -71,13 +66,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days):
 def _insert_inbound_sms_history(subquery, query_limit=10000):
     offset = 0
     inbound_sms_query = db.session.query(
-        InboundSms.id,
+        *[x.name for x in InboundSmsHistory.__table__.c]
-        InboundSms.created_at,
-        InboundSms.service_id,
-        InboundSms.notify_number,
-        InboundSms.provider_date,
-        InboundSms.provider_reference,
-        InboundSms.provider
     ).filter(InboundSms.id.in_(subquery))
     inbound_sms_count = inbound_sms_query.count()
 
@@ -119,7 +108,8 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
     return deleted
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def delete_inbound_sms_older_than_retention():
     current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 

app/dao/invited_org_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedOrganisationUser
 
 

app/dao/invited_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedUser
 
 
@@ -9,7 +9,7 @@ def save_invited_user(invited_user):
     db.session.commit()
 
 
-def get_invited_user_by_service_and_id(service_id, invited_user_id):
+def get_invited_user(service_id, invited_user_id):
     return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()
 
 

app/dao/jobs_dao.py

@@ -2,32 +2,37 @@ import uuid
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.letter_timings import (
+from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
-    CANCELLABLE_JOB_LETTER_STATUSES,
+from notifications_utils.statsd_decorators import statsd
-    letter_can_be_cancelled,
+from sqlalchemy import (
+    asc,
+    desc,
+    func,
+    and_
 )
-from sqlalchemy import and_, asc, desc, func
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.dao.templates_dao import dao_get_template_by_id
+from app.utils import midnight_n_days_ago
+
 from app.models import (
-    JOB_STATUS_CANCELLED,
+    Job,
     JOB_STATUS_FINISHED,
     JOB_STATUS_PENDING,
     JOB_STATUS_SCHEDULED,
     LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    NOTIFICATION_CREATED,
-    FactNotificationStatus,
-    Job,
     Notification,
-    ServiceDataRetention,
     Template,
+    ServiceDataRetention,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_CANCELLED,
+    JOB_STATUS_CANCELLED,
+    FactNotificationStatus
 )
-from app.utils import midnight_n_days_ago
 
 
+@statsd(namespace="dao")
 def dao_get_notification_outcomes_for_job(service_id, job_id):
     notification_statuses = db.session.query(
         func.count(Notification.status).label('count'), Notification.status
@@ -81,18 +86,6 @@ def dao_get_jobs_by_service_id(
         .paginate(page=page, per_page=page_size)
 
 
-def dao_get_scheduled_job_stats(
-    service_id,
-):
-    return db.session.query(
-        func.count(Job.id),
-        func.min(Job.scheduled_for),
-    ).filter(
-        Job.service_id == service_id,
-        Job.job_status == JOB_STATUS_SCHEDULED,
-    ).one()
-
-
 def dao_get_job_by_id(job_id):
     return Job.query.filter_by(id=job_id).one()
 
@@ -183,7 +176,7 @@ def dao_get_jobs_older_than_data_retention(notification_types):
     return jobs
 
 
-@autocommit
+@transactional
 def dao_cancel_letter_job(job):
     number_of_notifications_cancelled = Notification.query.filter(
         Notification.job_id == job.id
@@ -218,11 +211,12 @@ def can_letter_job_be_cancelled(job):
 
 
 def find_jobs_with_missing_rows():
-    # Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
+    # Jobs can be a maximum of 50,000 rows. It typically takes 5 minutes to create all those notifications.
-    # Using 20 minutes as a condition seems reasonable.
+    # Using 10 minutes as a condition seems reasonable.
-    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=20)
+    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=10)
     yesterday = datetime.utcnow() - timedelta(days=1)
     jobs_with_rows_missing = db.session.query(
+        func.count(Notification.id).label('actual_count'),
         Job
     ).filter(
         Job.job_status == JOB_STATUS_FINISHED,

app/dao/letter_branding_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import LetterBranding
 
 
@@ -15,12 +15,12 @@ def dao_get_all_letter_branding():
     return LetterBranding.query.order_by(LetterBranding.name).all()
 
 
-@autocommit
+@transactional
 def dao_create_letter_branding(letter_branding):
     db.session.add(letter_branding)
 
 
-@autocommit
+@transactional
 def dao_update_letter_branding(letter_branding_id, **kwargs):
     letter_branding = LetterBranding.query.get(letter_branding_id)
     for key, value in kwargs.items():

app/dao/notifications_dao.py

@@ -1,55 +1,60 @@
-from datetime import datetime, timedelta
+import functools
 from itertools import groupby
 from operator import attrgetter
+from datetime import (
+    datetime,
+    timedelta,
+)
 
 from botocore.exceptions import ClientError
 from flask import current_app
-from notifications_utils.international_billing_rates import (
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
-    INTERNATIONAL_BILLING_RATES,
-)
 from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
     validate_and_format_email_address,
+    InvalidEmailError,
+    try_validate_and_format_phone_number
 )
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
-from sqlalchemy import and_, asc, desc, func, or_, union
+from sqlalchemy import (desc, func, asc, and_, or_)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
 from werkzeug.datastructures import MultiDict
 
-from app import create_uuid, db, statsd_client
+from app import db, create_uuid
-from app.dao.dao_utils import autocommit
+from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
-from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
+from app.dao.dao_utils import transactional
+from app.letters.utils import get_letter_pdf_filename
 from app.models import (
-    EMAIL_TYPE,
+    FactNotificationStatus,
+    Notification,
+    NotificationHistory,
+    ProviderDetails,
     KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
     NOTIFICATION_PENDING,
     NOTIFICATION_PENDING_VIRUS_CHECK,
-    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
     NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENT,
     SMS_TYPE,
-    FactNotificationStatus,
+    EMAIL_TYPE,
-    Notification,
+    ServiceDataRetention,
-    NotificationHistory,
+    Service,
-    ProviderDetails,
-)
-from app.utils import (
-    escape_special_characters,
-    get_london_midnight_in_utc,
-    midnight_n_days_ago,
 )
+from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters
+from app.clients.sms.firetext import get_message_status_and_reason_from_firetext_code
 
 
+@statsd(namespace="dao")
 def dao_get_last_date_template_was_used(template_id, service_id):
     last_date_from_notifications = db.session.query(
         functions.max(Notification.created_at)
@@ -72,7 +77,8 @@ def dao_get_last_date_template_was_used(template_id, service_id):
     return last_date
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_create_notification(notification):
     if not notification.id:
         # need to populate defaulted fields before we create the notification history object
@@ -83,21 +89,41 @@ def dao_create_notification(notification):
     db.session.add(notification)
 
 
+def _decide_permanent_temporary_failure(status, notification, detailed_status_code=None):
+    # If we get failure status from Firetext, we want to know if this is temporary or permanent failure.
+    # So we check the failure code to learn that.
+    # If there is no failure code, or we do not recognise the failure code, we do the following:
+    # if notifitcation goes form status pending to status failure, we mark it as temporary failure;
+    # if notification goes straight to status failure, we mark it as permanent failure.
+    if status == NOTIFICATION_PERMANENT_FAILURE and detailed_status_code not in [None, '000']:
+        try:
+            status, reason = get_message_status_and_reason_from_firetext_code(detailed_status_code)
+            current_app.logger.info(f'Updating notification id {notification.id} to status {status}, reason: {reason}')
+            return status
+        except KeyError:
+            current_app.logger.warning(f'Failure code {detailed_status_code} from Firetext not recognised')
+    # fallback option:
+    if notification.status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+    return status
+
+
 def country_records_delivery(phone_prefix):
     dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
     return dlr and dlr.lower() == 'yes'
 
 
 def _update_notification_status(notification, status, detailed_status_code=None):
-    # status = _decide_permanent_temporary_failure(
+    status = _decide_permanent_temporary_failure(
-    #     status=status, notification=notification, detailed_status_code=detailed_status_code
+        status=status, notification=notification, detailed_status_code=detailed_status_code
-    # )
+    )
-    # notification.status = status
+    notification.status = status
-    # dao_update_notification(notification)
+    dao_update_notification(notification)
     return notification
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
 
@@ -118,11 +144,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
         _duplicate_update_warning(notification, status)
         return None
 
-    if (
+    if notification.international and not country_records_delivery(notification.phone_prefix):
-        notification.notification_type == SMS_TYPE
-        and notification.international
-        and not country_records_delivery(notification.phone_prefix)
-    ):
         return None
     if not notification.sent_by and sent_by:
         notification.sent_by = sent_by
@@ -133,7 +155,8 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
     )
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def update_notification_status_by_reference(reference, status):
     # this is used to update letters and emails
     notification = Notification.query.filter(Notification.reference == reference).first()
@@ -155,12 +178,19 @@ def update_notification_status_by_reference(reference, status):
     )
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_update_notification(notification):
     notification.updated_at = datetime.utcnow()
     db.session.add(notification)
 
 
+@statsd(namespace="dao")
+def get_notification_for_job(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
+@statsd(namespace="dao")
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -172,10 +202,12 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
     )
 
 
+@statsd(namespace="dao")
 def dao_get_notification_count_for_job_id(*, job_id):
     return Notification.query.filter_by(job_id=job_id).count()
 
 
+@statsd(namespace="dao")
 def get_notification_with_personalisation(service_id, notification_id, key_type):
     filter_dict = {'service_id': service_id, 'id': notification_id}
     if key_type:
@@ -184,6 +216,7 @@ def get_notification_with_personalisation(service_id, notification_id, key_type)
     return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one()
 
 
+@statsd(namespace="dao")
 def get_notification_by_id(notification_id, service_id=None, _raise=False):
     filters = [Notification.id == notification_id]
 
@@ -195,6 +228,11 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
     return query.one() if _raise else query.first()
 
 
+def get_notifications(filter_dict=None):
+    return _filter_query(Notification.query, filter_dict=filter_dict)
+
+
+@statsd(namespace="dao")
 def get_notifications_for_service(
         service_id,
         filter_dict=None,
@@ -208,8 +246,7 @@ def get_notifications_for_service(
         include_from_test_key=False,
         older_than=None,
         client_reference=None,
-        include_one_off=True,
+        include_one_off=True
-        error_out=True
 ):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -248,8 +285,7 @@ def get_notifications_for_service(
     return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
         per_page=page_size,
-        count=count_pages,
+        count=count_pages
-        error_out=error_out,
     )
 
 
@@ -273,42 +309,53 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
-@autocommit
+@statsd(namespace="dao")
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=50000):
+    current_app.logger.info(
+        'Deleting {} notifications for services with flexible data retention'.format(notification_type))
+
+    flexible_data_retention = ServiceDataRetention.query.filter(
+        ServiceDataRetention.notification_type == notification_type
+    ).all()
+    deleted = 0
+    for f in flexible_data_retention:
+        current_app.logger.info(
+            "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
+
+        day_to_delete_backwards_from = get_london_midnight_in_utc(
+            convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)
+
+        deleted += _move_notifications_to_notification_history(
+            notification_type, f.service_id, day_to_delete_backwards_from, qry_limit)
+
+    current_app.logger.info(
+        'Deleting {} notifications for services without flexible data retention'.format(notification_type))
+
+    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
+    services_with_data_retention = [x.service_id for x in flexible_data_retention]
+    service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
+
+    for service_id in service_ids_to_purge:
+        deleted += _move_notifications_to_notification_history(
+            notification_type, service_id, seven_days_ago, qry_limit)
+
+    current_app.logger.info('Finished deleting {} notifications'.format(notification_type))
+
+    return deleted
+
+
+@statsd(namespace="dao")
+@transactional
 def insert_notification_history_delete_notifications(
     notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
 ):
-    """
-    Delete up to 50,000 notifications that are past retention for a notification type and service.
-
-
-    Steps are as follows:
-
-    Create a temporary notifications table
-    Populate that table with up to 50k notifications that are to be deleted. (Note: no specified order)
-    Insert everything in the temp table into notification history
-    Delete from notifications if notification id is in the temp table
-    Drop the temp table (automatically when the transaction commits)
-
-    Temporary tables are in a separate postgres schema, and only visible to the current session (db connection,
-    in a celery task there's one connection per thread.)
-    """
     # Setting default query limit to 50,000 which take about 48 seconds on current table size
     # 10, 000 took 11s and 100,000 took 1 min 30 seconds.
-    select_into_temp_table = """
+    drop_table_if_exists = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
+        DROP TABLE if exists NOTIFICATION_ARCHIVE
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, postage, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
         """
-    select_into_temp_table_for_letters = """
+    select_into_temp_table = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
+         CREATE TEMP TABLE NOTIFICATION_ARCHIVE AS
          SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
              key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
              client_reference, international, phone_prefix, rate_multiplier, notification_status,
@@ -317,7 +364,6 @@ def insert_notification_history_delete_notifications(
         WHERE service_id = :service_id
           AND notification_type = :notification_type
           AND created_at < :timestamp_to_delete_backwards_from
-          AND notification_status NOT IN ('pending-virus-check', 'created', 'sending')
           AND key_type in ('normal', 'team')
         limit :qry_limit
         """
@@ -339,35 +385,31 @@ def insert_notification_history_delete_notifications(
         "qry_limit": qry_limit
     }
 
-    select_to_use = select_into_temp_table_for_letters if notification_type == 'letter' else select_into_temp_table
+    db.session.execute(drop_table_if_exists)
-    db.session.execute(select_to_use, input_params)
+    db.session.execute(select_into_temp_table, input_params)
 
-    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
+    result = db.session.execute("select * from NOTIFICATION_ARCHIVE")
 
     db.session.execute(insert_query)
 
     db.session.execute(delete_query)
 
-    return result
+    db.session.execute("DROP TABLE NOTIFICATION_ARCHIVE")
+    return result.rowcount
 
 
-def move_notifications_to_notification_history(
+def _move_notifications_to_notification_history(notification_type, service_id, day_to_delete_backwards_from, qry_limit):
-    notification_type,
-    service_id,
-    timestamp_to_delete_backwards_from,
-    qry_limit=50000
-):
     deleted = 0
     if notification_type == LETTER_TYPE:
         _delete_letters_from_s3(
-            notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit
+            notification_type, service_id, day_to_delete_backwards_from, qry_limit
         )
     delete_count_per_call = 1
     while delete_count_per_call > 0:
         delete_count_per_call = insert_notification_history_delete_notifications(
             notification_type=notification_type,
             service_id=service_id,
-            timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
+            timestamp_to_delete_backwards_from=day_to_delete_backwards_from,
             qry_limit=qry_limit
         )
         deleted += delete_count_per_call
@@ -376,7 +418,7 @@ def move_notifications_to_notification_history(
     Notification.query.filter(
         Notification.notification_type == notification_type,
         Notification.service_id == service_id,
-        Notification.created_at < timestamp_to_delete_backwards_from,
+        Notification.created_at < day_to_delete_backwards_from,
         Notification.key_type == KEY_TYPE_TEST
     ).delete(synchronize_session=False)
     db.session.commit()
@@ -392,55 +434,78 @@ def _delete_letters_from_s3(
     ).filter(
         Notification.notification_type == notification_type,
         Notification.created_at < date_to_delete_from,
-        Notification.service_id == service_id,
+        Notification.service_id == service_id
-        # although letters in non completed statuses do have PDFs in s3, they do not exist in the
-        # production-letters-pdf bucket as they never made it that far so we do not try and delete
-        # them from it
-        Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)
     ).limit(query_limit).all()
     for letter in letters_to_delete_from_s3:
-        try:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
-            letter_pdf = find_letter_pdf_in_s3(letter)
+        # I don't think we need this anymore, we should update the query to get letters sent 7 days ago
-            letter_pdf.delete()
+        if letter.sent_at:
-        except ClientError:
+            prefix = get_letter_pdf_filename(reference=letter.reference,
-            current_app.logger.exception(
+                                             crown=letter.service.crown,
-                "Error deleting S3 object for letter: {}".format(letter.id))
+                                             sending_date=letter.created_at,
-        except LetterPDFNotFound:
+                                             dont_use_sending_date=letter.key_type == KEY_TYPE_TEST,
-            current_app.logger.warning(
+                                             postage=letter.postage)
-                "No S3 object to delete for letter: {}".format(letter.id))
+            s3_objects = get_s3_bucket_objects(bucket_name=bucket_name, subfolder=prefix)
+            for s3_object in s3_objects:
+                try:
+                    remove_s3_object(bucket_name, s3_object['Key'])
+                except ClientError:
+                    current_app.logger.exception(
+                        "Could not delete S3 object with filename: {}".format(s3_object['Key']))
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_delete_notifications_by_id(notification_id):
     db.session.query(Notification).filter(
         Notification.id == notification_id
     ).delete(synchronize_session='fetch')
 
 
-def dao_timeout_notifications(cutoff_time, limit=100000):
+def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
-    """
-    Set email and SMS notifications (only) to "temporary-failure" status
-    if they're still sending from before the specified cutoff_time.
-    """
-    updated_at = datetime.utcnow()
-    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
-    new_status = NOTIFICATION_TEMPORARY_FAILURE
-
     notifications = Notification.query.filter(
-        Notification.created_at < cutoff_time,
+        Notification.created_at < timeout_start,
         Notification.status.in_(current_statuses),
-        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
+        Notification.notification_type != LETTER_TYPE
-    ).limit(limit).all()
+    ).all()
-
     Notification.query.filter(
-        Notification.id.in_([n.id for n in notifications]),
+        Notification.created_at < timeout_start,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type != LETTER_TYPE
     ).update(
         {'status': new_status, 'updated_at': updated_at},
         synchronize_session=False
     )
+    return notifications
+
+
+def dao_timeout_notifications(timeout_period_in_seconds):
+    """
+    Timeout SMS and email notifications by the following rules:
+
+    we never sent the notification to the provider for some reason
+        created -> technical-failure
+
+    the notification was sent to the provider but there was not a delivery receipt
+        sending -> temporary-failure
+        pending -> temporary-failure
+
+    Letter notifications are not timed out
+    """
+    timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
+    updated_at = datetime.utcnow()
+    timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
+
+    # Notifications still in created status are marked with a technical-failure:
+    technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
+
+    # Notifications still in sending or pending status are marked with a temporary-failure:
+    temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
+                                              NOTIFICATION_TEMPORARY_FAILURE)
 
     db.session.commit()
-    return notifications
+
+    return technical_failure_notifications, temporary_failure_notifications
 
 
 def is_delivery_slow_for_providers(
@@ -469,7 +534,6 @@ def is_delivery_slow_for_providers(
         ProviderDetails
     ).outerjoin(
         Notification, and_(
-            Notification.notification_type == SMS_TYPE,
             Notification.sent_by == ProviderDetails.identifier,
             Notification.created_at >= created_at,
             Notification.sent_at.isnot(None),
@@ -493,12 +557,16 @@ def is_delivery_slow_for_providers(
         slow_notifications = sum(row.count for row in rows if row.slow)
 
         slow_providers[provider] = (slow_notifications / total_notifications >= threshold)
-        statsd_client.gauge(f'slow-delivery.{provider}.ratio', slow_notifications / total_notifications)
+
+        current_app.logger.info("Slow delivery notifications count for provider {}: {} out of {}. Ratio {}".format(
+            provider, slow_notifications, total_notifications, slow_notifications / total_notifications
+        ))
 
     return slow_providers
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_update_notifications_by_reference(references, update_dict):
     updated_count = Notification.query.filter(
         Notification.reference.in_(references)
@@ -519,6 +587,7 @@ def dao_update_notifications_by_reference(references, update_dict):
     return updated_count, updated_history_count
 
 
+@statsd(namespace="dao")
 def dao_get_notifications_by_recipient_or_reference(
     service_id,
     search_term,
@@ -526,7 +595,6 @@ def dao_get_notifications_by_recipient_or_reference(
     statuses=None,
     page=1,
     page_size=None,
-    error_out=True,
 ):
 
     if notification_type == SMS_TYPE:
@@ -577,38 +645,46 @@ def dao_get_notifications_by_recipient_or_reference(
     results = db.session.query(Notification)\
         .filter(*filters)\
         .order_by(desc(Notification.created_at))\
-        .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
+        .paginate(page=page, per_page=page_size)
     return results
 
 
-def dao_get_notification_by_reference(reference):
+@statsd(namespace="dao")
+def dao_get_notification_by_reference(reference, notification_type):
     return Notification.query.filter(
-        Notification.reference == reference
+        Notification.reference == reference,
+        Notification.notification_type == notification_type
     ).one()
 
 
-def dao_get_notification_or_history_by_reference(reference):
+@statsd(namespace="dao")
+def dao_get_notification_or_history_by_reference(reference, notification_type):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object
         return Notification.query.filter(
-            Notification.reference == reference
+            Notification.reference == reference,
+            Notification.notification_type == notification_type
         ).one()
     except NoResultFound:
         return NotificationHistory.query.filter(
-            NotificationHistory.reference == reference
+            NotificationHistory.reference == reference,
+            NotificationHistory.notification_type == notification_type
         ).one()
 
 
-def dao_get_notifications_processing_time_stats(start_date, end_date):
+@statsd(namespace="dao")
-    """
+def dao_created_scheduled_notification(scheduled_notification):
-    For a given time range, returns the number of notifications sent and the number of
+    db.session.add(scheduled_notification)
-    those notifications that we processed within 10 seconds
+    db.session.commit()
 
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+    """
     SELECT
-    count(notifications),
+    count(notification_history),
     coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
     WHERE
     created_at > 'START DATE' AND
     created_at < 'END DATE' AND
@@ -638,6 +714,7 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
     ).one()
 
 
+@statsd(namespace="dao")
 def dao_get_last_notification_added_for_job_id(job_id):
     last_notification_added = Notification.query.filter(
         Notification.job_id == job_id
@@ -659,50 +736,17 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
+def dao_get_letters_to_be_printed(print_run_deadline):
     """
-    Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
+    Return all letters created before the print run deadline that have not yet been sent
-    to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
-    get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese  start sending off
-    tasks to the notify-ftp app to send them.
-
-    CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
-    there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
-    and we should avoid modifying state of returned objects.
-
-    For more reading:
-    https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
-    https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
     """
     notifications = Notification.query.filter(
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
+        Notification.key_type == KEY_TYPE_NORMAL
-        Notification.postage == postage,
-        Notification.billable_units > 0
     ).order_by(
-        Notification.service_id,
         Notification.created_at
-    ).yield_per(query_limit)
-    return notifications
-
-
-def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
-    notifications = db.session.query(
-        func.count(Notification.id).label('letters_count'),
-        func.sum(Notification.billable_units).label('sheets_count'),
-        Notification.postage
-    ).filter(
-        Notification.created_at < convert_bst_to_utc(print_run_deadline),
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.billable_units > 0
-    ).group_by(
-        Notification.postage
-    ).order_by(
-        Notification.postage
     ).all()
     return notifications
 
@@ -753,58 +797,14 @@ def dao_precompiled_letters_still_pending_virus_check():
 def _duplicate_update_warning(notification, status):
     current_app.logger.info(
         (
-            'Duplicate callback received for service {service_id}. '
+            'Duplicate callback received. Notification id {id} received a status update to {new_status}'
-            'Notification ID {id} with type {type} sent by {sent_by}. '
+            '{time_diff} after being set to {old_status}. {type} sent by {sent_by}'
-            'New status was {new_status}, current status is {old_status}. '
-            'This happened {time_diff} after being first set.'
         ).format(
             id=notification.id,
             old_status=notification.status,
             new_status=status,
             time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
             type=notification.notification_type,
-            sent_by=notification.sent_by,
+            sent_by=notification.sent_by
-            service_id=notification.service_id
         )
     )
-
-
-def get_service_ids_with_notifications_before(notification_type, timestamp):
-    return {
-        row.service_id
-        for row in db.session.query(
-            Notification.service_id
-        ).filter(
-            Notification.notification_type == notification_type,
-            Notification.created_at < timestamp
-        ).distinct()
-    }
-
-
-def get_service_ids_with_notifications_on_date(notification_type, date):
-    start_date = get_london_midnight_in_utc(date)
-    end_date = get_london_midnight_in_utc(date + timedelta(days=1))
-
-    notification_table_query = db.session.query(
-        Notification.service_id.label('service_id')
-    ).filter(
-        Notification.notification_type == notification_type,
-        # using >= + < is much more efficient than date(created_at)
-        Notification.created_at >= start_date,
-        Notification.created_at < end_date,
-    )
-
-    # Looking at this table is more efficient for historical notifications,
-    # provided the task to populate it has run before they were archived.
-    ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label('service_id')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.bst_date == date,
-    )
-
-    return {
-        row.service_id for row in db.session.query(union(
-            notification_table_query, ft_status_table_query
-        ).subquery()).distinct()
-    }

app/dao/organisation_dao.py

@@ -1,8 +1,14 @@
 from sqlalchemy.sql.expression import func
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
+from app.dao.dao_utils import VersionOptions, transactional, version_class
-from app.models import Domain, Organisation, Service, User
+from app.models import (
+    Organisation,
+    Domain,
+    InvitedOrganisationUser,
+    Service,
+    User
+)
 
 
 def dao_get_organisations():
@@ -55,12 +61,12 @@ def dao_get_organisation_by_service_id(service_id):
     return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()
 
 
-@autocommit
+@transactional
 def dao_create_organisation(organisation):
     db.session.add(organisation)
 
 
-@autocommit
+@transactional
 def dao_update_organisation(organisation_id, **kwargs):
 
     domains = kwargs.pop('domains', None)
@@ -83,9 +89,6 @@ def dao_update_organisation(organisation_id, **kwargs):
     if 'organisation_type' in kwargs:
         _update_organisation_services(organisation, 'organisation_type', only_where_none=False)
 
-    if 'crown' in kwargs:
-        _update_organisation_services(organisation, 'crown', only_where_none=False)
-
     if 'email_branding_id' in kwargs:
         _update_organisation_services(organisation, 'email_branding')
 
@@ -105,7 +108,7 @@ def _update_organisation_services(organisation, attribute, only_where_none=True)
         db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_add_service_to_organisation(service, organisation_id):
     organisation = Organisation.query.filter_by(
@@ -119,6 +122,10 @@ def dao_add_service_to_organisation(service, organisation_id):
     db.session.add(service)
 
 
+def dao_get_invited_organisation_user(user_id):
+    return InvitedOrganisationUser.query.filter_by(id=user_id).one()
+
+
 def dao_get_users_for_organisation(organisation_id):
     return db.session.query(
         User
@@ -130,15 +137,10 @@ def dao_get_users_for_organisation(organisation_id):
     ).order_by(User.created_at).all()
 
 
-@autocommit
+@transactional
 def dao_add_user_to_organisation(organisation_id, user_id):
     organisation = dao_get_organisation_by_id(organisation_id)
     user = User.query.filter_by(id=user_id).one()
     user.organisations.append(organisation)
     db.session.add(organisation)
     return user
-
-
-@autocommit
-def dao_remove_user_from_organisation(organisation, user):
-    organisation.users.remove(user)

app/dao/permissions_dao.py

@@ -1,16 +1,16 @@
 from app import db
 from app.dao import DAOClass
 from app.models import (
-    MANAGE_API_KEYS,
+    Permission,
-    MANAGE_SETTINGS,
-    MANAGE_TEMPLATES,
     MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
     SEND_EMAILS,
     SEND_LETTERS,
-    SEND_TEXTS,
+    MANAGE_API_KEYS,
-    VIEW_ACTIVITY,
+    VIEW_ACTIVITY)
-    Permission,
+
-)
 
 # Default permissions for a service
 default_service_permissions = [

app/dao/provider_details_dao.py

@@ -1,18 +1,12 @@
 from datetime import datetime, timedelta
 
-from flask import current_app
 from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import asc, desc, func
+from flask import current_app
 
+from app.dao.dao_utils import transactional
+from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    FactBilling,
-    ProviderDetails,
-    ProviderDetailsHistory,
-    User,
-)
 
 
 def get_provider_details_by_id(provider_details_id):
@@ -36,8 +30,6 @@ def dao_get_provider_versions(provider_id):
         id=provider_id
     ).order_by(
         desc(ProviderDetailsHistory.version)
-    ).limit(
-        100  # limit results instead of adding pagination
     ).all()
 
 
@@ -77,7 +69,7 @@ def _get_sms_providers_for_update(time_threshold):
     return q
 
 
-@autocommit
+@transactional
 def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     """
     Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
@@ -86,8 +78,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     amount_to_reduce_by = 10
     providers_list = _get_sms_providers_for_update(time_threshold)
 
-    if len(providers_list) < 2:
+    if not providers_list:
-        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
         return
 
     providers = {provider.identifier: provider for provider in providers_list}
@@ -104,7 +95,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     _adjust_provider_priority(increased_provider, increased_provider_priority)
 
 
-@autocommit
+@transactional
 def dao_adjust_provider_priority_back_to_resting_points():
     """
     Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
@@ -138,7 +129,7 @@ def get_provider_details_by_notification_type(notification_type, supports_intern
     return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()
 
 
-@autocommit
+@transactional
 def dao_update_provider_details(provider_details):
     _update_provider_details_without_commit(provider_details)
 

app/dao/provider_rates_dao.py

@@ -0,0 +1,11 @@
+from app.models import ProviderRates, ProviderDetails
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
+    db.session.add(provider_rates)

app/dao/returned_letters_dao.py

@@ -1,10 +1,10 @@
 from datetime import datetime
 
-from sqlalchemy import desc, func
+from sqlalchemy import func, desc
 from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import (
     Job,
     Notification,
@@ -28,7 +28,7 @@ def _get_notification_ids_for_references(references):
     return notification_ids + notification_history_ids
 
 
-@autocommit
+@transactional
 def insert_or_update_returned_letters(references):
     data = _get_notification_ids_for_references(references)
     for row in data:

app/dao/service_callback_api_dao.py

@@ -1,15 +1,13 @@
 from datetime import datetime
 
-from app import create_uuid, db
+from app import db, create_uuid
-from app.dao.dao_utils import autocommit, version_class
+from app.dao.dao_utils import transactional, version_class
-from app.models import (
+from app.models import ServiceCallbackApi
-    COMPLAINT_CALLBACK_TYPE,
+
-    DELIVERY_STATUS_CALLBACK_TYPE,
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE
-    ServiceCallbackApi,
-)
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
     service_callback_api.id = create_uuid()
@@ -17,7 +15,7 @@ def save_service_callback_api(service_callback_api):
     db.session.add(service_callback_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -48,6 +46,6 @@ def get_service_complaint_callback_api_for_service(service_id):
     ).first()
 
 
-@autocommit
+@transactional
 def delete_service_callback_api(service_callback_api):
     db.session.delete(service_callback_api)

app/dao/service_data_retention_dao.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceDataRetention
 
 
@@ -28,7 +28,7 @@ def fetch_service_data_retention_by_notification_type(service_id, notification_t
     return data_retention_list
 
 
-@autocommit
+@transactional
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
     new_data_retention = ServiceDataRetention(service_id=service_id,
                                               notification_type=notification_type,
@@ -38,7 +38,7 @@ def insert_service_data_retention(service_id, notification_type, days_of_retenti
     return new_data_retention
 
 
-@autocommit
+@transactional
 def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
     updated_count = ServiceDataRetention.query.filter(
         ServiceDataRetention.id == service_data_retention_id,
@@ -50,7 +50,3 @@ def update_service_data_retention(service_data_retention_id, service_id, days_of
         }
     )
     return updated_count
-
-
-def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
-    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()

app/dao/service_email_reply_to_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceEmailReplyTo
@@ -28,7 +28,7 @@ def dao_get_reply_to_by_id(service_id, reply_to_id):
     return reply_to
 
 
-@autocommit
+@transactional
 def add_reply_to_email_address_for_service(service_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -41,7 +41,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
     return new_reply_to
 
 
-@autocommit
+@transactional
 def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -57,7 +57,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
     return reply_to_update
 
 
-@autocommit
+@transactional
 def archive_reply_to_email_address(service_id, reply_to_id):
     reply_to_archive = ServiceEmailReplyTo.query.filter_by(
         id=reply_to_id,

app/dao/service_guest_list_dao.py

@@ -1,17 +0,0 @@
-from app import db
-from app.models import ServiceGuestList
-
-
-def dao_fetch_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).all()
-
-
-def dao_add_and_commit_guest_list_contacts(objs):
-    db.session.add_all(objs)
-    db.session.commit()
-
-
-def dao_remove_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).delete()

app/dao/service_inbound_api_dao.py

@@ -1,11 +1,11 @@
 from datetime import datetime
 
-from app import create_uuid, db
+from app import db, create_uuid
-from app.dao.dao_utils import autocommit, version_class
+from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceInboundApi
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
     service_inbound_api.id = create_uuid()
@@ -13,7 +13,7 @@ def save_service_inbound_api(service_inbound_api):
     db.session.add(service_inbound_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -35,6 +35,6 @@ def get_service_inbound_api_for_service(service_id):
     return ServiceInboundApi.query.filter_by(service_id=service_id).first()
 
 
-@autocommit
+@transactional
 def delete_service_inbound_api(service_inbound_api):
     db.session.delete(service_inbound_api)

app/dao/service_letter_contact_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceLetterContact, Template
 
 
@@ -30,7 +30,7 @@ def dao_get_letter_contact_by_id(service_id, letter_contact_id):
     return letter_contact
 
 
-@autocommit
+@transactional
 def add_letter_contact_for_service(service_id, contact_block, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -45,7 +45,7 @@ def add_letter_contact_for_service(service_id, contact_block, is_default):
     return new_letter_contact
 
 
-@autocommit
+@transactional
 def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
     old_default = _get_existing_default(service_id)
     # if we want to make this the default, ensure there are no other existing defaults
@@ -59,7 +59,7 @@ def update_letter_contact(service_id, letter_contact_id, contact_block, is_defau
     return letter_contact_update
 
 
-@autocommit
+@transactional
 def archive_letter_contact(service_id, letter_contact_id):
     letter_contact_to_archive = ServiceLetterContact.query.filter_by(
         id=letter_contact_id,

app/dao/service_permissions_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServicePermission
 
 
@@ -8,7 +8,7 @@ def dao_fetch_service_permissions(service_id):
         ServicePermission.service_id == service_id).all()
 
 
-@autocommit
+@transactional
 def dao_add_service_permission(service_id, permission):
     service_permission = ServicePermission(service_id=service_id, permission=permission)
     db.session.add(service_permission)

app/dao/service_sms_sender_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceSmsSender
 
@@ -32,7 +32,7 @@ def dao_get_sms_senders_by_service_id(service_id):
     ).order_by(desc(ServiceSmsSender.is_default)).all()
 
 
-@autocommit
+@transactional
 def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
     old_default = _get_existing_default(service_id=service_id)
     if is_default:
@@ -51,7 +51,7 @@ def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_n
     return new_sms_sender
 
 
-@autocommit
+@transactional
 def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -68,7 +68,7 @@ def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default,
     return sms_sender_to_update
 
 
-@autocommit
+@transactional
 def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
     service_sms_sender.sms_sender = sms_sender
     service_sms_sender.inbound_number_id = inbound_number_id
@@ -76,7 +76,7 @@ def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sende
     return service_sms_sender
 
 
-@autocommit
+@transactional
 def archive_sms_sender(service_id, sms_sender_id):
     sms_sender_to_archive = ServiceSmsSender.query.filter_by(
         id=sms_sender_id,

app/dao/service_user_dao.py

@@ -1,6 +1,6 @@
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceUser, User
 
 
@@ -9,13 +9,9 @@ def dao_get_service_user(user_id, service_id):
 
 
 def dao_get_active_service_users(service_id):
-    query = db.session.query(
+    query = ServiceUser.query.join(ServiceUser.user).filter(
-        ServiceUser
+        ServiceUser.service_id == service_id,
-    ).join(
+        User.state == 'active'
-        User, User.id == ServiceUser.user_id
-    ).filter(
-        User.state == 'active',
-        ServiceUser.service_id == service_id
     )
 
     return query.all()
@@ -25,6 +21,6 @@ def dao_get_service_users_by_user_id(user_id):
     return ServiceUser.query.filter_by(user_id=user_id).all()
 
 
-@autocommit
+@transactional
 def dao_update_service_user(service_user):
     db.session.add(service_user)

app/dao/service_whitelist_dao.py

@@ -0,0 +1,17 @@
+from app import db
+from app.models import ServiceWhitelist
+
+
+def dao_fetch_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).all()
+
+
+def dao_add_and_commit_whitelisted_contacts(objs):
+    db.session.add_all(objs)
+    db.session.commit()
+
+
+def dao_remove_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).delete()

app/dao/services_dao.py

@@ -1,14 +1,19 @@
 import uuid
 from datetime import date, datetime, timedelta
 
-from flask import current_app
+from notifications_utils.statsd_decorators import statsd
-from sqlalchemy import Float, cast
+from sqlalchemy.sql.expression import asc, case, and_, func
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import and_, asc, case, func
+from sqlalchemy import cast, Float
+from flask import current_app
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
 from app.dao.email_branding_dao import dao_get_email_branding_by_name
 from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
@@ -16,17 +21,6 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    CROWN_ORGANISATION_TYPES,
-    EMAIL_TYPE,
-    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_SMS_TYPE,
-    KEY_TYPE_TEST,
-    LETTER_TYPE,
-    NHS_ORGANISATION_TYPES,
-    NON_CROWN_ORGANISATION_TYPES,
-    NOTIFICATION_PERMANENT_FAILURE,
-    SMS_TYPE,
-    UPLOAD_LETTERS,
     AnnualBilling,
     ApiKey,
     FactBilling,
@@ -38,22 +32,33 @@ from app.models import (
     Organisation,
     Permission,
     Service,
-    ServiceContactList,
-    ServiceEmailReplyTo,
-    ServiceLetterContact,
     ServicePermission,
     ServiceSmsSender,
+    ServiceEmailReplyTo,
+    ServiceContactList,
+    ServiceLetterContact,
     Template,
     TemplateHistory,
     TemplateRedacted,
     User,
     VerifyCode,
+    CROWN_ORGANISATION_TYPES,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NHS_ORGANISATION_TYPES,
+    NON_CROWN_ORGANISATION_TYPES,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    UPLOAD_LETTERS,
 )
 from app.utils import (
     email_address_is_nhs,
     escape_special_characters,
     get_archived_db_column_value,
     get_london_midnight_in_utc,
+    midnight_n_days_ago,
 )
 
 DEFAULT_SERVICE_PERMISSIONS = [
@@ -62,7 +67,6 @@ DEFAULT_SERVICE_PERMISSIONS = [
     LETTER_TYPE,
     INTERNATIONAL_SMS_TYPE,
     UPLOAD_LETTERS,
-    INTERNATIONAL_LETTERS,
 ]
 
 
@@ -246,7 +250,7 @@ def dao_fetch_all_services_created_by_user(user_id):
     return query.all()
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -283,7 +287,7 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
     ).one()
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_create_service(
     service,
@@ -291,6 +295,9 @@ def dao_create_service(
     service_id=None,
     service_permissions=None,
 ):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.
 
     if not user:
         raise ValueError("Can't create a service without a user")
@@ -317,17 +324,15 @@ def dao_create_service(
     if organisation:
         service.organisation_id = organisation.id
         service.organisation_type = organisation.organisation_type
-
         if organisation.email_branding:
             service.email_branding = organisation.email_branding
 
-        if organisation.letter_branding:
+        if organisation.letter_branding and not service.letter_branding:
             service.letter_branding = organisation.letter_branding
 
     elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
         service.email_branding = dao_get_email_branding_by_name('NHS')
         service.letter_branding = dao_get_letter_branding_by_name('NHS')
-
     if organisation:
         service.crown = organisation.crown
     elif service.organisation_type in CROWN_ORGANISATION_TYPES:
@@ -339,7 +344,7 @@ def dao_create_service(
     db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_update_service(service):
     db.session.add(service)
@@ -421,24 +426,51 @@ def delete_service_and_all_associated_db_objects(service):
     db.session.commit()
 
 
-def dao_fetch_todays_stats_for_service(service_id):
+@statsd(namespace="dao")
-    today = date.today()
+def dao_fetch_stats_for_service(service_id, limit_days):
-    start_date = get_london_midnight_in_utc(today)
+    # We always want between seven and eight days
+    start_date = midnight_n_days_ago(limit_days)
+    return _stats_for_service_query(service_id).filter(
+        Notification.created_at >= start_date
+    ).all()
 
+
+@statsd(namespace="dao")
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def fetch_todays_total_message_count(service_id):
+    result = db.session.query(
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        func.date(Notification.created_at) == date.today()
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).first()
+    return 0 if result is None else result.count
+
+
+def _stats_for_service_query(service_id):
     return db.session.query(
         Notification.notification_type,
         Notification.status,
         func.count(Notification.id).label('count')
     ).filter(
         Notification.service_id == service_id,
-        Notification.key_type != KEY_TYPE_TEST,
+        Notification.key_type != KEY_TYPE_TEST
-        Notification.created_at >= start_date
     ).group_by(
         Notification.notification_type,
         Notification.status,
-    ).all()
+    )
 
 
+@statsd(namespace='dao')
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
     today = date.today()
     start_date = get_london_midnight_in_utc(today)
@@ -484,7 +516,7 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act
     return query.all()
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -503,7 +535,7 @@ def dao_suspend_service(service_id):
     service.active = False
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_resume_service(service_id):
     service = Service.query.get(service_id)
@@ -531,8 +563,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
         Notification.notification_type == SMS_TYPE,
         func.substr(Notification.normalised_to, 3, 7) == '7700900',
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
+        Service.research_mode == False,
-        Service.active == True,  # noqa
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -540,7 +572,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
     ).all()
 
 
-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
     subquery = db.session.query(
         func.count(Notification.id).label('total_count'),
         Notification.service_id.label('service_id')
@@ -551,8 +583,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.key_type != KEY_TYPE_TEST,
         Notification.notification_type == SMS_TYPE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
+        Service.research_mode == False,
-        Service.active == True,  # noqa
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -577,8 +609,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.notification_type == SMS_TYPE,
         Notification.status == NOTIFICATION_PERMANENT_FAILURE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
+        Service.research_mode == False,
-        Service.active == True,  # noqa
+        Service.active == True,
     ).group_by(
         Notification.service_id,
         subquery.c.total_count
@@ -587,23 +619,3 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
     )
 
     return query.all()
-
-
-def get_live_services_with_organisation():
-    query = db.session.query(
-        Service.id.label("service_id"),
-        Service.name.label("service_name"),
-        Organisation.id.label("organisation_id"),
-        Organisation.name.label("organisation_name")
-    ).outerjoin(
-        Service.organisation
-    ).filter(
-        Service.count_as_live.is_(True),
-        Service.active.is_(True),
-        Service.restricted.is_(False)
-    ).order_by(
-        Organisation.name,
-        Service.name
-    )
-
-    return query.all()

app/dao/template_folder_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import TemplateFolder
 
 
@@ -14,16 +14,16 @@ def dao_get_valid_template_folders_by_id(folder_ids):
     return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all()
 
 
-@autocommit
+@transactional
 def dao_create_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_update_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_delete_template_folder(template_folder):
     db.session.delete(template_folder)

app/dao/templates_dao.py

@@ -1,22 +1,26 @@
-import uuid
 from datetime import datetime
+import uuid
 
 from flask import current_app
 from sqlalchemy import asc, desc
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.users_dao import get_user_by_id
 from app.models import (
     LETTER_TYPE,
     SECOND_CLASS,
     Template,
     TemplateHistory,
-    TemplateRedacted,
+    TemplateRedacted
 )
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.users_dao import get_user_by_id
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
@@ -38,15 +42,18 @@ def dao_create_template(template):
     db.session.add(template)
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
 def dao_update_template(template):
+    if template.archived:
+        template.folder = None
+
     db.session.add(template)
 
 
-@autocommit
+@transactional
 def dao_update_template_reply_to(template_id, reply_to):
     Template.query.filter_by(id=template_id).update(
         {"service_letter_contact_id": reply_to,
@@ -71,14 +78,13 @@ def dao_update_template_reply_to(template_id, reply_to):
                                   "version": template.version,
                                   "archived": template.archived,
                                   "process_type": template.process_type,
-                                  "service_letter_contact_id": template.service_letter_contact_id,
+                                  "service_letter_contact_id": template.service_letter_contact_id
-                                  "broadcast_data": template.broadcast_data,
                               })
     db.session.add(history)
     return template
 
 
-@autocommit
+@transactional
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True
     template.template_redacted.updated_at = datetime.utcnow()

app/dao/uploads_dao.py

@@ -1,18 +1,11 @@
 from datetime import datetime
-
 from flask import current_app
-from sqlalchemy import String, and_, desc, func, literal, text
+from sqlalchemy import and_, desc, func, literal, text, String
 
 from app import db
 from app.models import (
-    JOB_STATUS_CANCELLED,
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
-    JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED, ServiceDataRetention
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    Job,
-    Notification,
-    ServiceDataRetention,
-    Template,
 )
 from app.utils import midnight_n_days_ago
 
@@ -50,8 +43,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
         func.coalesce(
             Job.processing_started, Job.created_at
-        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
+        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
-        Job.contact_list_id.is_(None),
     ]
     if limit_days is not None:
         jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
@@ -84,7 +76,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
         Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
+        Template.hidden == True,
         Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:

app/dao/users_dao.py

@@ -1,16 +1,16 @@
+from random import (SystemRandom)
+from datetime import (datetime, timedelta)
 import uuid
-from datetime import datetime, timedelta
-from random import SystemRandom
 
 from sqlalchemy import func
 from sqlalchemy.orm import joinedload
 
 from app import db
-from app.dao.dao_utils import autocommit
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_users_by_user_id
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
-from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
+from app.models import (EMAIL_AUTH_TYPE, User, VerifyCode)
 from app.utils import escape_special_characters, get_archived_db_column_value
 
 
@@ -20,23 +20,15 @@ def _remove_values_for_keys_if_present(dict, keys):
 
 
 def create_secret_code():
-    return ''.join(get_non_repeating_random_digits(5))
+    return ''.join(map(str, [SystemRandom().randrange(10) for i in range(5)]))
 
 
-def get_non_repeating_random_digits(length):
+def save_user_attribute(usr, update_dict={}):
-    output = [None] * length
+    db.session.query(User).filter_by(id=usr.id).update(update_dict)
-    for index in range(length):
-        while output[index] in {None, output[index - 1]}:
-            output[index] = str(SystemRandom().randrange(10))
-    return output
-
-
-def save_user_attribute(usr, update_dict=None):
-    db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
     db.session.commit()
 
 
-def save_model_user(user, update_dict=None, password=None, validated_email_access=False):
+def save_model_user(user, update_dict={}, password=None, validated_email_access=False):
     if password:
         user.password = password
         user.password_changed_at = datetime.utcnow()
@@ -44,7 +36,7 @@ def save_model_user(user, update_dict=None, password=None, validated_email_acces
         user.email_access_validated_at = datetime.utcnow()
     if update_dict:
         _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
-        db.session.query(User).filter_by(id=user.id).update(update_dict or {})
+        db.session.query(User).filter_by(id=user.id).update(update_dict)
     else:
         db.session.add(user)
     db.session.commit()
@@ -131,10 +123,12 @@ def reset_failed_login_count(user):
         db.session.commit()
 
 
-def update_user_password(user, password):
+def update_user_password(user, password, validated_email_access=False):
     # reset failed login count - they've just reset their password so should be fine
     user.password = password
     user.password_changed_at = datetime.utcnow()
+    if validated_email_access:
+        user.email_access_validated_at = datetime.utcnow()
     db.session.add(user)
     db.session.commit()
 
@@ -152,7 +146,7 @@ def get_user_and_accounts(user_id):
     ).one()
 
 
-@autocommit
+@transactional
 def dao_archive_user(user):
     if not user_can_be_archived(user):
         msg = "User can’t be removed from a service - check all services have another team member with manage_settings"

app/dao/webauthn_credential_dao.py

@@ -1,40 +0,0 @@
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import WebauthnCredential
-
-
-def dao_get_webauthn_credential_by_user_and_id(user_id, webauthn_credential_id):
-    return WebauthnCredential.query.filter(
-        WebauthnCredential.user_id == user_id,
-        WebauthnCredential.id == webauthn_credential_id
-    ).one()
-
-
-@autocommit
-def dao_create_webauthn_credential(
-    *,
-    user_id,
-    name,
-    credential_data,
-    registration_response,
-):
-    webauthn_credential = WebauthnCredential(
-        user_id=user_id,
-        name=name,
-        credential_data=credential_data,
-        registration_response=registration_response
-    )
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_update_webauthn_credential_name(webauthn_credential, new_name):
-    webauthn_credential.name = new_name
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_delete_webauthn_credential(webauthn_credential):
-    db.session.delete(webauthn_credential)

app/delivery/send_to_providers.py

@@ -1,43 +1,39 @@
 import random
-from datetime import datetime, timedelta
 from urllib import parse
+from datetime import datetime, timedelta
 
-from cachetools import TTLCache, cached
 from flask import current_app
-from notifications_utils.template import (
+from notifications_utils.recipients import (
-    HTMLEmailTemplate,
+    validate_and_format_phone_number,
-    PlainTextEmailTemplate,
+    validate_and_format_email_address
-    SMSMessageTemplate,
 )
+from notifications_utils.template import HTMLEmailTemplate, PlainTextEmailTemplate, SMSMessageTemplate
 
-from app import create_uuid, db, notification_provider_clients, statsd_client
+from app import clients, statsd_client, create_uuid
-from app.celery.research_mode_tasks import (
+from app.dao.notifications_dao import (
-    send_email_response,
+    dao_update_notification
-    send_sms_response,
 )
-from app.dao.email_branding_dao import dao_get_email_branding_by_id
-from app.dao.notifications_dao import dao_update_notification
 from app.dao.provider_details_dao import (
-    dao_reduce_sms_provider_priority,
     get_provider_details_by_notification_type,
+    dao_reduce_sms_provider_priority
 )
+from app.celery.research_mode_tasks import send_sms_response, send_email_response
+from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
+    SMS_TYPE,
+    KEY_TYPE_TEST,
     BRANDING_BOTH,
     BRANDING_ORG_BANNER,
     EMAIL_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
     NOTIFICATION_TECHNICAL_FAILURE,
-    SMS_TYPE,
+    NOTIFICATION_SENT,
+    NOTIFICATION_SENDING
 )
-from app.serialised_models import SerialisedService, SerialisedTemplate
 
 
 def send_sms_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
+    service = notification.service
 
     if not service.active:
         technical_failure(notification=notification)
@@ -45,13 +41,8 @@ def send_sms_to_provider(notification):
 
     if notification.status == 'created':
         provider = provider_to_use(SMS_TYPE, notification.international)
-        if not provider:
-            technical_failure(notification=notification)
-            return
 
-        template_model = SerialisedTemplate.from_id_and_service_id(
+        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        )
 
         template = SMSMessageTemplate(
             template_model.__dict__,
@@ -59,63 +50,41 @@ def send_sms_to_provider(notification):
             prefix=service.name,
             show_prefix=service.prefix_sms,
         )
-        created_at = notification.created_at
+
-        key_type = notification.key_type
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.get_name(), str(notification.id), notification.to)
 
         else:
             try:
-                # End DB session here so that we don't have a connection stuck open waiting on the call
+                provider.send_sms(
-                # to one of the SMS providers
+                    to=validate_and_format_phone_number(notification.to, international=notification.international),
-                # We don't want to tie our DB connections being open to the performance of our SMS
+                    content=str(template),
-                # providers as a slow down of our providers can cause us to run out of DB connections
+                    reference=str(notification.id),
-                # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
+                    sender=notification.reply_to_text
-                # closing the session (as otherwise it would be reopened immediately)
+                )
-                send_sms_kwargs = {
-                    'to': notification.normalised_to,
-                    'content': str(template),
-                    'reference': str(notification.id),
-                    'sender': notification.reply_to_text,
-                    'international': notification.international,
-                }
-                db.session.close()  # no commit needed as no changes to objects have been made above
-                provider.send_sms(**send_sms_kwargs)
             except Exception as e:
                 notification.billable_units = template.fragment_count
                 dao_update_notification(notification)
-                dao_reduce_sms_provider_priority(provider.name, time_threshold=timedelta(minutes=1))
+                dao_reduce_sms_provider_priority(provider.get_name(), time_threshold=timedelta(minutes=1))
                 raise e
             else:
                 notification.billable_units = template.fragment_count
                 update_notification_to_sending(notification, provider)
 
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
         statsd_client.timing("sms.total-time", delta_seconds)
 
-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("sms.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("sms.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("sms.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("sms.live-key.not-high-volume.total-time", delta_seconds)
-
 
 def send_email_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
+    service = notification.service
-
     if not service.active:
         technical_failure(notification=notification)
         return
     if notification.status == 'created':
         provider = provider_to_use(EMAIL_TYPE)
 
-        template_dict = SerialisedTemplate.from_id_and_service_id(
+        template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        ).__dict__
 
         html_email = HTMLEmailTemplate(
             template_dict,
@@ -127,8 +96,7 @@ def send_email_to_provider(notification):
             template_dict,
             values=notification.personalisation
         )
-        created_at = notification.created_at
+
-        key_type = notification.key_type
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             notification.reference = str(create_uuid())
             update_notification_to_sending(notification, provider)
@@ -137,45 +105,33 @@ def send_email_to_provider(notification):
             from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
                                                  current_app.config['NOTIFY_EMAIL_DOMAIN'])
 
+            email_reply_to = notification.reply_to_text
+
             reference = provider.send_email(
                 from_address,
-                notification.normalised_to,
+                validate_and_format_email_address(notification.to),
                 plain_text_email.subject,
                 body=str(plain_text_email),
                 html_body=str(html_email),
-                reply_to_address=notification.reply_to_text
+                reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
             )
             notification.reference = reference
             update_notification_to_sending(notification, provider)
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
 
-        if key_type == KEY_TYPE_TEST:
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
-            statsd_client.timing("email.test-key.total-time", delta_seconds)
+        statsd_client.timing("email.total-time", delta_seconds)
-        else:
-            statsd_client.timing("email.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
 
 
 def update_notification_to_sending(notification, provider):
     notification.sent_at = datetime.utcnow()
-    notification.sent_by = provider.name
+    notification.sent_by = provider.get_name()
-    if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
+    notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
-        notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
     dao_update_notification(notification)
 
 
-provider_cache = TTLCache(maxsize=8, ttl=10)
+def provider_to_use(notification_type, international=False):
-
-
-@cached(cache=provider_cache)
-def provider_to_use(notification_type, international=True):
-    international = False # TODO: remove or resolve the functionality of this flag
-    # TODO rip firetext and mmg out of early migrations and clean up the expression below
     active_providers = [
-        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg']
+        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active
     ]
 
     if not active_providers:
@@ -184,13 +140,9 @@ def provider_to_use(notification_type, international=True):
         )
         raise Exception("No active {} providers".format(notification_type))
 
-    if len(active_providers) == 1:
+    chosen_provider = random.choices(active_providers, weights=[p.priority for p in active_providers])[0]
-        chosen_provider = active_providers[0]
-    else:
-        weights = [p.priority for p in active_providers]
-        chosen_provider = random.choices(active_providers, weights=weights)[0]
 
-    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
+    return clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
 
 
 def get_logo_url(base_url, logo_file):
@@ -215,28 +167,25 @@ def get_logo_url(base_url, logo_file):
 
 
 def get_html_email_options(service):
+
     if service.email_branding is None:
         return {
             'govuk_banner': True,
             'brand_banner': False,
         }
-    if isinstance(service, SerialisedService):
-        branding = dao_get_email_branding_by_id(service.email_branding)
-    else:
-        branding = service.email_branding
 
     logo_url = get_logo_url(
         current_app.config['ADMIN_BASE_URL'],
-        branding.logo
+        service.email_branding.logo
-    ) if branding.logo else None
+    ) if service.email_branding.logo else None
 
     return {
-        'govuk_banner': branding.brand_type == BRANDING_BOTH,
+        'govuk_banner': service.email_branding.brand_type == BRANDING_BOTH,
-        'brand_banner': branding.brand_type == BRANDING_ORG_BANNER,
+        'brand_banner': service.email_branding.brand_type == BRANDING_ORG_BANNER,
-        'brand_colour': branding.colour,
+        'brand_colour': service.email_branding.colour,
         'brand_logo': logo_url,
-        'brand_text': branding.text,
+        'brand_text': service.email_branding.text,
-        'brand_name': branding.name,
+        'brand_name': service.email_branding.name,
     }
 
 

app/email_branding/email_branding_schema.py

@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES
 
 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }
 
 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {

app/email_branding/rest.py

@@ -2,16 +2,16 @@ from flask import Blueprint, jsonify, request
 
 from app.dao.email_branding_dao import (
     dao_create_email_branding,
-    dao_get_email_branding_by_id,
     dao_get_email_branding_options,
-    dao_update_email_branding,
+    dao_get_email_branding_by_id,
-)
+    dao_update_email_branding
-from app.email_branding.email_branding_schema import (
-    post_create_email_branding_schema,
-    post_update_email_branding_schema,
 )
 from app.errors import register_errors
 from app.models import EmailBranding
+from app.email_branding.email_branding_schema import (
+    post_create_email_branding_schema,
+    post_update_email_branding_schema
+)
 from app.schema_validation import validate
 
 email_branding_blueprint = Blueprint('email_branding', __name__)

app/errors.py

@@ -1,10 +1,12 @@
-from flask import current_app, json, jsonify
+from flask import (
-from jsonschema import ValidationError as JsonSchemaValidationError
+    jsonify,
-from marshmallow import ValidationError
+    current_app,
+    json)
 from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
-
+from marshmallow import ValidationError
+from jsonschema import ValidationError as JsonSchemaValidationError
 from app.authentication.auth import AuthError
 from app.exceptions import ArchiveValidationError
 

app/events/rest.py

@@ -1,8 +1,13 @@
-from flask import Blueprint, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)
 
-from app.dao.events_dao import dao_create_event
 from app.errors import register_errors
+
 from app.schemas import event_schema
+from app.dao.events_dao import dao_create_event
 
 events = Blueprint('events', __name__, url_prefix='/events')
 register_errors(events)
@@ -11,6 +16,6 @@ register_errors(events)
 @events.route('', methods=['POST'])
 def create_event():
     data = request.get_json()
-    event = event_schema.load(data)
+    event = event_schema.load(data).data
     dao_create_event(event)
-    return jsonify(data=event_schema.dump(event)), 201
+    return jsonify(data=event_schema.dump(event).data), 201

app/govuk_alerts/rest.py

@@ -1,31 +0,0 @@
-from flask import Blueprint, jsonify
-
-from app.dao.broadcast_message_dao import dao_get_all_broadcast_messages
-from app.errors import register_errors
-from app.utils import get_dt_string_or_none
-
-govuk_alerts_blueprint = Blueprint(
-    "govuk-alerts",
-    __name__,
-    url_prefix='/govuk-alerts',
-)
-
-register_errors(govuk_alerts_blueprint)
-
-
-@govuk_alerts_blueprint.route('')
-def get_broadcasts():
-    broadcasts = dao_get_all_broadcast_messages()
-    broadcasts_dict = {"alerts": [{
-        "id": broadcast.id,
-        "reference": broadcast.reference,
-        "channel": broadcast.channel,
-        "content": broadcast.content,
-        "areas": broadcast.areas,
-        "status": broadcast.status,
-        "starts_at": get_dt_string_or_none(broadcast.starts_at),
-        "finishes_at": get_dt_string_or_none(broadcast.finishes_at),
-        "approved_at": get_dt_string_or_none(broadcast.approved_at),
-        "cancelled_at": get_dt_string_or_none(broadcast.cancelled_at),
-    } for broadcast in broadcasts]}
-    return jsonify(broadcasts_dict), 200

app/hashing.py

@@ -1,4 +1,4 @@
-from flask_bcrypt import check_password_hash, generate_password_hash
+from flask_bcrypt import generate_password_hash, check_password_hash
 
 
 def hashpw(password):

app/history_meta.py

@@ -15,11 +15,11 @@ session events.
 
 """
 import datetime
-
-from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, util
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import attributes, mapper, object_mapper
+from sqlalchemy.orm import mapper, attributes, object_mapper
-from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty
+from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty
+from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
+from sqlalchemy import util
 
 
 def col_references_table(col, table):

app/inbound_number/rest.py

@@ -1,10 +1,10 @@
 from flask import Blueprint, jsonify
 
 from app.dao.inbound_numbers_dao import (
-    dao_get_available_inbound_numbers,
-    dao_get_inbound_number_for_service,
     dao_get_inbound_numbers,
-    dao_set_inbound_number_active_flag,
+    dao_get_inbound_number_for_service,
+    dao_get_available_inbound_numbers,
+    dao_set_inbound_number_active_flag
 )
 from app.errors import register_errors
 

app/inbound_sms/inbound_sms_schemas.py

@@ -1,5 +1,5 @@
 get_inbound_sms_for_service_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "schema for parameters allowed when searching for to field=",
     "type": "object",
     "properties": {