Some small changes to letter tasks and adding statsd

- Add statsd to collate-letter-pdfs-to-be-sent - Add statsd to dao_get_letters_to_be_printed - rename scheduled task for clarity
2026-08-21 06:49:26 -04:00 · 2020-03-11 17:24:15 +00:00
459 changed files with 14483 additions and 28699 deletions
--- a/.flake8
+++ b/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
+# W503: line break before binary operator
+exclude = venv*,__pycache__,node_modules,cache,migrations,build
+ignore = W503
+max-complexity = 14
+max-line-length = 120
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -1,76 +0,0 @@
-name: Run checks
-
-on: [push]
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  ANTIVIRUS_ENABLED: 0
-  NOTIFY_ENVIRONMENT: development
-  NOTIFICATION_QUEUE_PREFIX: local_dev_10x
-  STATSD_HOST: localhost
-  SES_STUB_URL: None
-  NOTIFY_APP_NAME: api
-  NOTIFY_EMAIL_DOMAIN: dispostable.com
-  NOTIFY_LOG_PATH: /workspace/logs/app.log
-  ADMIN_CLIENT_ID: notify-admin
-  ADMIN_CLIENT_SECRET: dev-notify-secret-key
-  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
-  FLASK_APP: application.py
-  FLASK_ENV: development
-  WERKZEUG_DEBUG_PIN: off
-  ADMIN_BASE_URL: http://localhost:6012
-  API_HOST_NAME: http://localhost:6011
-  REDIS_URL: redis://localhost:6380
-  REDIS_ENABLED: False
-  SQLALCHEMY_DATABASE_URI: postgresql://postgres:chummy@db:5432/notification_api
-  SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-  AWS_REGION: us-west-2
-  AWS_PINPOINT_REGION: us-west-2
-  AWS_US_TOLL_FREE_NUMBER: +18446120782
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
-    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-    - name: Install application dependencies
-      run: make bootstrap
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-    # - name: Run style checks
-    #   run: flake8 .
-    # - name: Check imports alphabetized
-    #   run: isort --check-only ./app ./tests
-    - name: Run tests
-      run: pytest -n4 --maxfail=10
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,58 +0,0 @@
-name: Deploy to prototype environment
-
-on:
-  workflow_run:
-    workflows: [ Run checks ]
-    types:
-      - completed
-    branches: [ main ] # Redundant, workflow_run events are only triggered on default branch (`main`)
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-
-    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-
-    - uses: actions/checkout@v3
-
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-
-    - name: Install application dependencies
-      run: make bootstrap
-
-    - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
-      env:
-        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      with:
-        cf_username: ${{ secrets.cloudgov_username }}
-        cf_password: ${{ secrets.cloudgov_password }}
-        cf_org: gsa-10x-prototyping
-        cf_space: 10x-notifications
-        full_command: |
-          cf push --strategy rolling \
-          --var DANGEROUS_SALT="$DANGEROUS_SALT" \
-          --var SECRET_KEY="$SECRET_KEY" \
-          --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
-          --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
-
-  bail:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - run: echo 'Checks failed, not deploying'
--- a/.gitignore
+++ b/.gitignore
@@ -69,15 +69,10 @@ target/
 *.DS_Store
 environment.sh
 .envrc
-.env
-.env*
-varsfile

 celerybeat-schedule

 # CloudFoundry
 .cf
-varsfile*
-.secret*

 /scripts/run_my_tests.sh
--- a/.pyup.yml
+++ b/.pyup.yml
@@ -4,5 +4,5 @@ schedule: "every week on wednesday"

 search: False
 requirements:
-  - requirements.in
-  - requirements_for_test.txt
+  - requirements-app.txt
+  - requirements-dev.txt
--- a/166
+++ b/166
@@ -7,79 +7,102 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)

+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
+
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
+
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
+
 CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)

-CF_MANIFEST_PATH ?= /tmp/manifest.yml
-
+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml

 NOTIFY_CREDENTIALS ?= ~/.notify-credentials


 ## DEVELOPMENT

-.PHONY: bootstrap
-bootstrap: generate-version-file ## Set up everything to run the app
-	pip3 install -r requirements_for_test.txt
-	createdb notification_api || true
-	(flask db upgrade) || true
-
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
-
-.PHONY: run-flask
-run-flask: ## Run flask
-	flask run -p 6011 --host=0.0.0.0
-
-.PHONY: run-celery
-run-celery: ## Run celery, TODO remove purge for staging/prod
-	celery -A run_celery.notify_celery purge -f
-	celery \
-		-A run_celery.notify_celery worker \
-		--pidfile="/tmp/celery.pid" \
-		--loglevel=INFO \
-		--concurrency=4
-
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
-.PHONY: run-celery-beat
-run-celery-beat: ## Run celery beat
-	celery \
-	-A run_celery.notify_celery beat \
-	--loglevel=INFO
-
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
-	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
+	@echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE}

 .PHONY: test
-test: ## Run tests
-	# flake8 .
-	isort --check-only ./app ./tests
-	pytest -n4 --maxfail=10
+test: generate-version-file ## Run tests
+	./scripts/run_tests.sh

 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pip install --upgrade pip-tools
-	pip-compile requirements.in
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
+
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
+
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
+	docker build -f docker/Dockerfile \
+		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
+		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
+		--build-arg NO_PROXY="${NO_PROXY}" \
+		-t ${DOCKER_BUILDER_IMAGE_NAME} \
+		.
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	@docker run -it --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
+		-e GIT_COMMIT=${GIT_COMMIT} \
+		-e BUILD_NUMBER=${BUILD_NUMBER} \
+		-e BUILD_URL=${BUILD_URL} \
+		-e http_proxy="${HTTP_PROXY}" \
+		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e https_proxy="${HTTPS_PROXY}" \
+		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e NO_PROXY="${NO_PROXY}" \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: create-docker-test-db
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	@docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true

 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
+	rm -rf node_modules cache target venv .coverage build tests/.cache


 ## DEPLOYMENT
@@ -87,19 +110,16 @@ clean:
 .PHONY: preview
 preview: ## Set environment to preview
 	$(eval export DEPLOY_ENV=preview)
-	$(eval export DNS_NAME="notify.works")
 	@true

 .PHONY: staging
 staging: ## Set environment to staging
 	$(eval export DEPLOY_ENV=staging)
-	$(eval export DNS_NAME="staging-notify.works")
 	@true

 .PHONY: production
 production: ## Set environment to production
 	$(eval export DEPLOY_ENV=production)
-	$(eval export DNS_NAME="notifications.service.gov.uk")
 	@true

 .PHONY: cf-login
@@ -131,56 +151,28 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 	@cf app --guid ${CF_APP} || exit 1

 	# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
-	cf cancel-deployment ${CF_APP} || true
+	cf v3-cancel-zdt-push ${CF_APP} || true
+
+	cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
+	CF_STARTUP_TIMEOUT=10 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete  # fails after 5 mins if deploy doesn't work

-	# fails after 15 mins if deploy doesn't work
-	CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling

 .PHONY: cf-deploy-api-db-migration
 cf-deploy-api-db-migration:
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
-	make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH}
-
-	cf push notifications-api --no-route -f ${CF_MANIFEST_PATH}
-	rm ${CF_MANIFEST_PATH}
-
-	cf run-task notifications-api --command="flask db upgrade" --name api_db_migration
+	cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
+	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration

 .PHONY: cf-check-api-db-migration-task
-cf-check-api-db-migration-task: ## Get the status for the last notifications-api task
-	@cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state"
+cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
+	@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"

 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
 	$(if ${CF_APP},,$(error Must specify CF_APP))
-	rm ${CF_MANIFEST_PATH}
-	cf cancel-deployment ${CF_APP}
+	cf v3-cancel-zdt-push ${CF_APP}

 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:
 	@echo $(shell python3 scripts/check_if_new_migration.py)
-
-.PHONY: cf-deploy-failwhale
-cf-deploy-failwhale:
-	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
-	cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
-
-.PHONY: enable-failwhale
-enable-failwhale: ## Enable the failwhale app and disable api
-	$(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`))
-	# make sure failwhale is running first
-	cf start notify-api-failwhale
-
-	cf map-route notify-api-failwhale ${DNS_NAME} --hostname api
-	cf unmap-route notify-api ${DNS_NAME} --hostname api
-	@echo "Failwhale is enabled"
-
-.PHONY: disable-failwhale
-disable-failwhale: ## Disable the failwhale app and enable api
-	$(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`))
-
-	cf map-route notify-api ${DNS_NAME} --hostname api
-	cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
-	cf stop notify-api-failwhale
-	@echo "Failwhale is disabled"
--- a/3
+++ b/3
@@ -1,2 +1 @@
-web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
-worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null
+web: ./scripts/paas_app_wrapper.sh
--- a/README.md
+++ b/README.md
@@ -1,142 +1,158 @@
-# US Notify API
-
-Cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
+# GOV.UK Notify API

 Contains:
-
- the public-facing REST API for US Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation) [DOCS ARE STILL UK]
- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/18F/notifications-admin) talks to)
+- the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation)
+- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to)
 - asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc

-## QUICKSTART
---
-If you are the first on your team to deploy, set up AWS SES/SNS as instructed in the AWS setup section below.
-
-Create .env file as described in the .env section below.
-
-Install VS Code
-Open VS Code and install the Remote-Containers plug-in from Microsoft.
-
-Make sure your docker daemon is running (on OS X, this is typically accomplished by opening the Docker Desktop app)
-
-Create the external docker network:
-
-`docker network create notify-network`
-
-Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...”
-When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). 
-
-After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes.
-
-Select View->Open View..., then search/select “ports”. Await a green dot on the port view, then open a new terminal and run the web server:
-`make run-flask`
-
-Open another terminal and run the background tasks:
-`make run-celery`
-
---
 ## Setting Up

-### `.env` file
+### Python version

-Create and edit a .env file, based on sample.env. 
+At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.

-NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
+### AWS credentials

-Things to change:
+To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.

- If you're not the first to deploy, only replace the aws creds, get these from team lead
- Replace `NOTIFICATION_QUEUE_PREFIX` with `local_dev_<your org>_`
- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project)
- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values
- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars
+Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.

-### AWS Setup
-
-**Steps to prepare SES**
-
-1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
-2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
-
-TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures
-
-**Steps to prepare SNS**
-
-1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms"
-2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save
-3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers"
-4. Under "Number Settings" choose "Request Phone Number"
-5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request"
-6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)"
-7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number)
-
-At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
-
-### Secrets Detection
+### Virtualenv

 ```
-brew install detect-secrets # or pip install detect-secrets
-detect-secrets scan
-#review output of above, make sure none of the baseline entries are sensitive
-detect-secrets scan > .secrets.baseline
-#creates the baseline file
+mkvirtualenv -p /usr/local/bin/python3 notifications-api
 ```

-Ideally, you'll install `detect-secrets` so that it's accessible from any environment from which you _might_ commit. You can use `brew install` to make it available globally. You could also install via `pip install` inside a virtual environment, if you're sure you'll _only_ commit from that environment.
+### `environment.sh`

-If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test).
+Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).

-You can permit exceptions by adding an inline comment containing `pragma: allowlist secret`
+Create a local environment.sh file containing the following:

-The command that is actually run by the pre-commit hook is: `git diff --staged --name-only -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+```
+echo "
+export NOTIFY_ENVIRONMENT='development'

-You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+export MMG_API_KEY='MMG_API_KEY'
+export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
+export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
+
+export FLASK_APP=application.py
+export FLASK_DEBUG=1
+export WERKZEUG_DEBUG_PIN=off
+"> environment.sh
+```
+
+NOTES:
+
+ * Replace the placeholder key and prefix values as appropriate
+ * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
+ * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.

 ### Postgres

-Local postgres implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
+
+Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.

 ### Redis

-Local redis implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:

-## To test the application
+        REDIS_ENABLED = True
+
+
+##  To run the application
+
+First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
+
+You need to run the api application and a local celery instance.
+
+There are two run scripts for running all the necessary parts.

 ```
-# install dependencies, etc.
-make bootstrap
+scripts/run_app.sh
+```

+```
+scripts/run_celery.sh
+```
+
+Optionally you can also run this script to run the scheduled tasks:
+
+```
+scripts/run_celery_beat.sh
+```
+
+
+##  To test the application
+
+First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
+
+Then simply run
+
+```
 make test
 ```

-## To run scheduled tasks
+That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
+[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
+
+
+## To update application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with

 ```
-# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat 
-make run-celery-beat
+make freeze-requirements
 ```

-## To run one off tasks (Ignore for Quick Start)
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.
+
+
+## To run one off tasks

 Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
 care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
 example, to purge all dynamically generated functional test data, do the following:

-Local (from inside the devcontainer)
-
+Locally
 ```
 flask command purge_functional_test_data -u <functional tests user name prefix>
 ```

-Remote
-
+On the server
 ```
 cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
 ```

 All commands and command options have a --help command if you need more information.

-## Further documentation [DEPRECATED]

- [Writing public APIs](docs/writing-public-apis.md)
- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
+## To create a new worker app
+
+You need to:
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
+1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
+1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
+1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
+
+**Important:**
+
+Before pushing the deployment change on jenkins, read below about the first time deployment.
+
+### First time deployment of your new worker
+
+Our deployment flow requires that the app is present in order to proceed with the deployment.
+
+This means that the first deployment of your app must happen manually.
+
+To do this:
+
+1. Ensure your code is backwards compatible
+1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
+
+Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.
--- a/app/init.py
+++ b/app/init.py
@@ -1,40 +1,32 @@
 import os
 import random
 import string
-import time
 import uuid
-from time import monotonic

-from celery import current_task
-from flask import (
-    current_app,
-    g,
-    has_request_context,
-    jsonify,
-    make_response,
-    request,
-)
+from flask import _request_ctx_stack, request, g, jsonify, make_response
+from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
-from gds_metrics import GDSMetrics
-from gds_metrics.metrics import Gauge, Histogram
-from notifications_utils import logging, request_helper
-from notifications_utils.celery import NotifyCelery
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from time import monotonic
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
-from sqlalchemy import event
+from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils import logging, request_helper
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy

-from app.clients import NotificationProviderClients
-from app.clients.cbc_proxy import CBCProxyClient
+from app.celery.celery import NotifyCelery
+from app.clients import Clients
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
-from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.aws_sns import AwsSnsClient
+from app.clients.sms.firetext import FiretextClient
+from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+
+DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+DATE_FORMAT = "%Y-%m-%d"


 class SQLAlchemy(_SQLAlchemy):
@@ -53,26 +45,20 @@ db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
+firetext_client = FiretextClient()
+mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
-aws_ses_stub_client = AwsSesStubClient()
-aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
-cbc_proxy_client = CBCProxyClient()
+performance_platform_client = PerformancePlatformClient()
 document_download_client = DocumentDownloadClient()
-metrics = GDSMetrics()

-notification_provider_clients = NotificationProviderClients()
+clients = Clients()

-api_user = LocalProxy(lambda: g.api_user)
-authenticated_service = LocalProxy(lambda: g.authenticated_service)
-
-CONCURRENT_REQUESTS = Gauge(
-    'concurrent_web_request_count',
-    'How many concurrent requests are currently being served',
-)
+api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
+authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)


 def create_app(application):
@@ -84,9 +70,6 @@ def create_app(application):

    application.config['NOTIFY_APP_NAME'] = application.name
    init_app(application)
-
-    # Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded
-    metrics.init_app(application)
    request_helper.init_app(application)
    db.init_app(application)
    migrate.init_app(application, db=db)
@@ -94,27 +77,15 @@ def create_app(application):
    zendesk_client.init_app(application)
    statsd_client.init_app(application)
    logging.init_app(application, statsd_client)
-    aws_sns_client.init_app(application, statsd_client=statsd_client)
-
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application, statsd_client=statsd_client)
    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
-    aws_ses_stub_client.init_app(
-        application.config['AWS_REGION'],
-        statsd_client=statsd_client,
-        stub_url=application.config['SES_STUB_URL']
-    )
-    # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
-    email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
-    notification_provider_clients.init_app(
-        sms_clients=[aws_sns_client],
-        email_clients=email_clients
-    )
-
    notify_celery.init_app(application)
    encryption.init_app(application)
    redis_store.init_app(application)
+    performance_platform_client.init_app(application)
    document_download_client.init_app(application)
-
-    cbc_proxy_client.init_app(application)
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client])

    register_blueprint(application)
    register_v2_blueprints(application)
@@ -123,63 +94,38 @@ def create_app(application):
    from app.commands import setup_commands
    setup_commands(application)

-    # set up sqlalchemy events
-    setup_sqlalchemy_events(application)
-
    return application


 def register_blueprint(application):
-    from app.authentication.auth import (
-        requires_admin_auth,
-        requires_auth,
-        requires_govuk_alerts_auth,
-        requires_no_auth,
-    )
-    from app.billing.rest import billing_blueprint
-    from app.broadcast_message.rest import broadcast_message_blueprint
-    from app.complaint.complaint_rest import complaint_blueprint
-    from app.email_branding.rest import email_branding_blueprint
+    from app.service.rest import service_blueprint
+    from app.service.callback_rest import service_callback_blueprint
+    from app.user.rest import user_blueprint
+    from app.template.rest import template_blueprint
+    from app.status.healthcheck import status as status_blueprint
+    from app.job.rest import job_blueprint
+    from app.notifications.rest import notifications as notifications_blueprint
+    from app.invite.rest import invite as invite_blueprint
+    from app.accept_invite.rest import accept_invite
+    from app.template_statistics.rest import template_statistics as template_statistics_blueprint
    from app.events.rest import events as events_blueprint
-    from app.govuk_alerts.rest import govuk_alerts_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.email_branding.rest import email_branding_blueprint
    from app.inbound_number.rest import inbound_number_blueprint
    from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
-    from app.job.rest import job_blueprint
-    from app.letter_branding.letter_branding_rest import (
-        letter_branding_blueprint,
-    )
+    from app.notifications.receive_notifications import receive_notifications_blueprint
+    from app.notifications.notifications_sms_callback import sms_callback_blueprint
+    from app.notifications.notifications_letter_callback import letter_callback_blueprint
+    from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
    from app.letters.rest import letter_job
-    from app.notifications.notifications_letter_callback import (
-        letter_callback_blueprint,
-    )
-    from app.notifications.notifications_sms_callback import (
-        sms_callback_blueprint,
-    )
-    from app.notifications.receive_notifications import (
-        receive_notifications_blueprint,
-    )
-    from app.notifications.rest import notifications as notifications_blueprint
-    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.billing.rest import billing_blueprint
    from app.organisation.rest import organisation_blueprint
-    from app.performance_dashboard.rest import performance_dashboard_blueprint
+    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.complaint.complaint_rest import complaint_blueprint
    from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import (
-        provider_details as provider_details_blueprint,
-    )
-    from app.service.callback_rest import service_callback_blueprint
-    from app.service.rest import service_blueprint
-    from app.service_invite.rest import (
-        service_invite as service_invite_blueprint,
-    )
-    from app.status.healthcheck import status as status_blueprint
-    from app.template.rest import template_blueprint
    from app.template_folder.rest import template_folder_blueprint
-    from app.template_statistics.rest import (
-        template_statistics as template_statistics_blueprint,
-    )
+    from app.letter_branding.letter_branding_rest import letter_branding_blueprint
    from app.upload.rest import upload_blueprint
-    from app.user.rest import user_blueprint
-    from app.webauthn.rest import webauthn_blueprint

    service_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -187,9 +133,6 @@ def register_blueprint(application):
    user_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(user_blueprint, url_prefix='/user')

-    webauthn_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(webauthn_blueprint)
-
    template_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(template_blueprint)

@@ -211,11 +154,8 @@ def register_blueprint(application):
    job_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(job_blueprint)

-    service_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_invite_blueprint)
-
-    organisation_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organisation_invite_blueprint)
+    invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(invite_blueprint)

    inbound_number_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(inbound_number_blueprint)
@@ -223,6 +163,9 @@ def register_blueprint(application):
    inbound_sms_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(inbound_sms_blueprint)

+    accept_invite.before_request(requires_admin_auth)
+    application.register_blueprint(accept_invite, url_prefix='/invite')
+
    template_statistics_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(template_statistics_blueprint)

@@ -250,12 +193,12 @@ def register_blueprint(application):
    organisation_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(organisation_blueprint, url_prefix='/organisations')

+    organisation_invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(organisation_invite_blueprint)
+
    complaint_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(complaint_blueprint)

-    performance_dashboard_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(performance_dashboard_blueprint)
-
    platform_stats_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')

@@ -268,58 +211,47 @@ def register_blueprint(application):
    upload_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(upload_blueprint)

-    broadcast_message_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(broadcast_message_blueprint)
-
-    govuk_alerts_blueprint.before_request(requires_govuk_alerts_auth)
-    application.register_blueprint(govuk_alerts_blueprint)
-

 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
+    from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
+    from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
+    from app.v2.template.get_template import v2_template_blueprint as get_template
+    from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
+    from app.v2.template.post_template import v2_template_blueprint as post_template
    from app.authentication.auth import requires_auth
-    from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint

-    v2_notification_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
+    post_notifications.before_request(requires_auth)
+    application.register_blueprint(post_notifications)

-    v2_templates_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
+    get_notifications.before_request(requires_auth)
+    application.register_blueprint(get_notifications)

-    v2_template_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
+    get_templates.before_request(requires_auth)
+    application.register_blueprint(get_templates)

-    v2_inbound_sms_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
+    get_template.before_request(requires_auth)
+    application.register_blueprint(get_template)

-    v2_broadcast_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_broadcast_blueprint)
+    post_template.before_request(requires_auth)
+    application.register_blueprint(post_template)
+
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)


 def init_app(app):
+    @app.before_request
+    def record_user_agent():
+        statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))

    @app.before_request
    def record_request_details():
-        CONCURRENT_REQUESTS.inc()
-
        g.start = monotonic()
        g.endpoint = request.endpoint

    @app.after_request
    def after_request(response):
-        CONCURRENT_REQUESTS.dec()
-
        response.headers.add('Access-Control-Allow-Origin', '*')
        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
@@ -355,87 +287,13 @@ def create_random_identifier():
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))


-def setup_sqlalchemy_events(app):
-
-    TOTAL_DB_CONNECTIONS = Gauge(
-        'db_connection_total_connected',
-        'How many db connections are currently held (potentially idle) by the server',
-    )
-
-    TOTAL_CHECKED_OUT_DB_CONNECTIONS = Gauge(
-        'db_connection_total_checked_out',
-        'How many db connections are currently checked out by web requests',
-    )
-
-    DB_CONNECTION_OPEN_DURATION_SECONDS = Histogram(
-        'db_connection_open_duration_seconds',
-        'How long db connections are held open for in seconds',
-        ['method', 'host', 'path']
-    )
-
-    # need this or db.engine isn't accessible
-    with app.app_context():
-        @event.listens_for(db.engine, 'connect')
-        def connect(dbapi_connection, connection_record):
-            # connection first opened with db
-            TOTAL_DB_CONNECTIONS.inc()
-
-        @event.listens_for(db.engine, 'close')
-        def close(dbapi_connection, connection_record):
-            # connection closed (probably only happens with overflow connections)
-            TOTAL_DB_CONNECTIONS.dec()
-
-        @event.listens_for(db.engine, 'checkout')
-        def checkout(dbapi_connection, connection_record, connection_proxy):
-            try:
-                # connection given to a web worker
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.inc()
-
-                # this will overwrite any previous checkout_at timestamp
-                connection_record.info['checkout_at'] = time.monotonic()
-
-                # checkin runs after the request is already torn down, therefore we add the request_data onto the
-                # connection_record as otherwise it won't have that information when checkin actually runs.
-                # Note: this is not a problem for checkouts as the checkout always happens within a web request or task
-
-                # web requests
-                if has_request_context():
-                    connection_record.info['request_data'] = {
-                        'method': request.method,
-                        'host': request.host,
-                        'url_rule': request.url_rule.rule if request.url_rule else 'No endpoint'
-                    }
-                # celery apps
-                elif current_task:
-                    connection_record.info['request_data'] = {
-                        'method': 'celery',
-                        'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
-                        'url_rule': current_task.name,  # task name
-                    }
-                # anything else. migrations possibly, or flask cli commands.
-                else:
-                    current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
-                    connection_record.info['request_data'] = {
-                        'method': 'unknown',
-                        'host': 'unknown',
-                        'url_rule': 'unknown',
-                    }
-            except Exception:
-                current_app.logger.exception("Exception caught for checkout event.")
-
-        @event.listens_for(db.engine, 'checkin')
-        def checkin(dbapi_connection, connection_record):
-            try:
-                # connection returned by a web worker
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.dec()
-
-                # duration that connection was held by a single web request
-                duration = time.monotonic() - connection_record.info['checkout_at']
-
-                DB_CONNECTION_OPEN_DURATION_SECONDS.labels(
-                    connection_record.info['request_data']['method'],
-                    connection_record.info['request_data']['host'],
-                    connection_record.info['request_data']['url_rule']
-                ).observe(duration)
-            except Exception:
-                current_app.logger.exception("Exception caught for checkin event.")
+def process_user_agent(user_agent_string):
+    if user_agent_string and user_agent_string.lower().startswith("notify"):
+        components = user_agent_string.split("/")
+        client_name = components[0].lower()
+        client_version = components[1].replace(".", "-")
+        return "{}.{}".format(client_name, client_version)
+    elif user_agent_string and not user_agent_string.lower().startswith("notify"):
+        return "non-notify-user-agent"
+    else:
+        return "unknown"
--- a/app/broadcast_message/init.py
+++ b/app/broadcast_message/init.py
--- a/app/accept_invite/rest.py
+++ b/app/accept_invite/rest.py
@@ -0,0 +1,52 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    current_app
+)
+
+from itsdangerous import SignatureExpired, BadData
+
+from notifications_utils.url_safe_token import check_token
+
+from app.dao.invited_user_dao import get_invited_user_by_id
+from app.dao.organisation_dao import dao_get_invited_organisation_user
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
+from app.schemas import invited_user_schema
+
+
+accept_invite = Blueprint('accept_invite', __name__)
+register_errors(accept_invite)
+
+
+@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
+def validate_invitation_token(invitation_type, token):
+
+    max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
+
+    try:
+        invited_user_id = check_token(token,
+                                      current_app.config['SECRET_KEY'],
+                                      current_app.config['DANGEROUS_SALT'],
+                                      max_age_seconds)
+    except SignatureExpired:
+        errors = {'invitation':
+                  ['Your invitation to GOV.UK Notify has expired. '
+                   'Please ask the person that invited you to send you another one']}
+        raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
+
+    if invitation_type == 'service':
+        invited_user = get_invited_user_by_id(invited_user_id)
+        return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+    elif invitation_type == 'organisation':
+        invited_user = dao_get_invited_organisation_user(invited_user_id)
+        return jsonify(data=invited_user.serialize()), 200
+    else:
+        raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))
--- a/app/authentication/auth.py
+++ b/app/authentication/auth.py
@@ -1,30 +1,17 @@
-import uuid
-
-from flask import current_app, g, request
-from gds_metrics import Histogram
-from notifications_python_client.authentication import (
-    decode_jwt_token,
-    get_token_issuer,
-)
+from flask import request, _request_ctx_stack, current_app, g
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
 from notifications_python_client.errors import (
-    TokenAlgorithmError,
-    TokenDecodeError,
-    TokenError,
-    TokenExpiredError,
-    TokenIssuerError,
+    TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError, TokenError
 )
 from notifications_utils import request_helper
+from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound

-from app.serialised_models import SerialisedService
+from app.dao.services_dao import dao_fetch_service_by_id_with_api_keys
+

 GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa

-AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
-    'auth_db_connection_duration_seconds',
-    'Time taken to get DB connection and fetch service from database',
-)
-

 class AuthError(Exception):
    def __init__(self, message, code, service_id=None, api_key_id=None):
@@ -49,115 +36,7 @@ class AuthError(Exception):
        }


-class InternalApiKey():
-    def __init__(self, client_id, secret):
-        self.secret = secret
-        self.id = client_id
-        self.expiry_date = None
-
-
-def requires_no_auth():
-    pass
-
-
-def requires_govuk_alerts_auth():
-    requires_internal_auth(current_app.config.get('GOVUK_ALERTS_CLIENT_ID'))
-
-
-def requires_admin_auth():
-    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
-
-
-def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
-        raise TypeError("Unknown client_id for internal auth")
-
-    request_helper.check_proxy_header_before_request()
-    auth_token = _get_auth_token(request)
-    client_id = _get_token_issuer(auth_token)
-
-    if client_id != expected_client_id:
-        current_app.logger.info('client_id: %s', client_id)
-        current_app.logger.info('expected_client_id: %s', expected_client_id)
-        raise AuthError("Unauthorized: not allowed to perform this action", 401)
-
-    api_keys = [
-        InternalApiKey(client_id, secret)
-        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
-    ]
-
-    _decode_jwt_token(auth_token, api_keys, client_id)
-    g.service_id = client_id
-
-
-def requires_auth():
-    request_helper.check_proxy_header_before_request()
-
-    auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
-
-    try:
-        service_id = uuid.UUID(issuer)
-    except Exception:
-        raise AuthError("Invalid token: service id is not the right data type", 403)
-
-    try:
-        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
-            service = SerialisedService.from_id(service_id)
-    except NoResultFound:
-        raise AuthError("Invalid token: service not found", 403)
-
-    if not service.api_keys:
-        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
-
-    if not service.active:
-        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
-
-    api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
-
-    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
-        service_id,
-        api_key.id,
-        request.headers.get('User-Agent'),
-        request.base_url
-    ))
-
-    g.api_user = api_key
-    g.service_id = service_id
-    g.authenticated_service = service
-
-
-def _decode_jwt_token(auth_token, api_keys, service_id=None):
-    for api_key in api_keys:
-        try:
-            decode_jwt_token(auth_token, api_key.secret)
-        except TokenExpiredError:
-            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenAlgorithmError:
-            err_msg = "Invalid token: algorithm used is not HS256"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenDecodeError:
-            # we attempted to validate the token but it failed meaning it was not signed using this api key.
-            # Let's try the next one
-            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
-            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
-            # continue on to check the next API key
-            continue
-        except TokenError:
-            # General error when trying to decode and validate the token
-            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
-
-        if api_key.expiry_date:
-            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
-
-        return api_key
-    else:
-        # service has API keys, but none matching the one the user provided
-        raise AuthError("Invalid token: API key not found", 403, service_id=service_id)
-
-
-def _get_auth_token(req):
+def get_auth_token(req):
    auth_header = req.headers.get('Authorization', None)
    if not auth_header:
        raise AuthError('Unauthorized: authentication token must be provided', 401)
@@ -170,7 +49,96 @@ def _get_auth_token(req):
    return auth_header[7:]


-def _get_token_issuer(auth_token):
+def requires_no_auth():
+    pass
+
+
+def requires_admin_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
+
+    if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
+        g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
+
+        for secret in current_app.config.get('API_INTERNAL_SECRETS'):
+            try:
+                decode_jwt_token(auth_token, secret)
+                return
+            except TokenExpiredError:
+                raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
+            except TokenDecodeError:
+                # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions
+                # (which are children of `TokenDecodeError`) as these should cause an auth error immediately rather
+                # than continue on to check the next admin client secret
+                continue
+
+        # Either there are no admin client secrets or their token didn't match one of them so error
+        raise AuthError("Unauthorized: admin authentication token not found", 401)
+    else:
+        raise AuthError('Unauthorized: admin authentication token required', 401)
+
+
+def requires_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    issuer = __get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
+
+    try:
+        service = dao_fetch_service_by_id_with_api_keys(issuer)
+    except DataError:
+        raise AuthError("Invalid token: service id is not the right data type", 403)
+    except NoResultFound:
+        raise AuthError("Invalid token: service not found", 403)
+
+    if not service.api_keys:
+        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
+
+    if not service.active:
+        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
+
+    for api_key in service.api_keys:
+        try:
+            decode_jwt_token(auth_token, api_key.secret)
+        except TokenExpiredError:
+            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenAlgorithmError:
+            err_msg = "Invalid token: algorithm used is not HS256"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenDecodeError:
+            # we attempted to validate the token but it failed meaning it was not signed using this api key.
+            # Let's try the next one
+            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
+            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
+            # continue on to check the next API key
+            continue
+        except TokenError:
+            # General error when trying to decode and validate the token
+            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service.id, api_key_id=api_key.id)
+
+        if api_key.expiry_date:
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
+
+        g.service_id = api_key.service_id
+        _request_ctx_stack.top.authenticated_service = service
+        _request_ctx_stack.top.api_user = api_key
+
+        current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
+            service.id,
+            api_key.id,
+            request.headers.get('User-Agent'),
+            request.base_url
+        ))
+        return
+    else:
+        # service has API keys, but none matching the one the user provided
+        raise AuthError("Invalid token: API key not found", 403, service_id=service.id)
+
+
+def __get_token_issuer(auth_token):
    try:
        issuer = get_token_issuer(auth_token)
    except TokenIssuerError:
--- a/app/aws/s3.py
+++ b/app/aws/s3.py
@@ -1,7 +1,11 @@
-import botocore
-from boto3 import client, resource
+from datetime import datetime, timedelta
+
 from flask import current_app

+import pytz
+from boto3 import client, resource
+import botocore
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'


@@ -39,13 +43,6 @@ def get_job_location(service_id, job_id):
    )


-def get_contact_list_location(service_id, contact_list_id):
-    return (
-        current_app.config['CONTACT_LIST_BUCKET_NAME'],
-        FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
-    )
-
-
 def get_job_and_metadata_from_s3(service_id, job_id):
    obj = get_s3_object(*get_job_location(service_id, job_id))
    return obj.get()['Body'].read().decode('utf-8'), obj.get()['Metadata']
@@ -65,8 +62,40 @@ def remove_job_from_s3(service_id, job_id):
    return remove_s3_object(*get_job_location(service_id, job_id))


-def remove_contact_list_from_s3(service_id, contact_list_id):
-    return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))
+def get_s3_bucket_objects(bucket_name, subfolder=''):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        if page.get('Contents'):
+            all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
+
+
+def filter_s3_bucket_objects_within_date_range(bucket_objects, older_than=7, limit_days=2):
+    """
+    S3 returns the Object['LastModified'] as an 'offset-aware' timestamp so the
+    date range filter must take this into account.
+
+    Additionally an additional Object is returned by S3 corresponding to the
+    container directory. This is redundant and should be removed.
+
+    """
+    end_date = datetime.now(tz=pytz.utc) - timedelta(days=older_than)
+    start_date = end_date - timedelta(days=limit_days)
+    filtered_items = [item for item in bucket_objects if all([
+        not item['Key'].endswith('/'),
+        item['LastModified'] > start_date,
+        item['LastModified'] < end_date
+    ])]
+
+    return filtered_items


 def remove_s3_object(bucket_name, object_key):
--- a/app/billing/billing_schemas.py
+++ b/app/billing/billing_schemas.py
@@ -1,45 +1,42 @@
 from datetime import datetime

+
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST annual billing schema",
    "type": "object",
    "title": "Create",
    "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
    },
    "required": ["free_sms_fragment_limit"]
 }


-def serialize_ft_billing_remove_emails(rows):
-    return [
-        {
-            "month": (datetime.strftime(row.month, "%B")),
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "postage": row.postage,
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_remove_emails(data):
+    results = []
+    billed_notifications = [x for x in data if x.notification_type != 'email']
+    for notification in billed_notifications:
+        json_result = {
+            "month": (datetime.strftime(notification.month, "%B")),
+            "notification_type": notification.notification_type,
+            "billing_units": notification.billable_units,
+            "rate": float(notification.rate),
+            "postage": notification.postage,
        }
-        for row in rows
-        if row.notification_type != 'email'
-    ]
+        results.append(json_result)
+    return results


-def serialize_ft_billing_yearly_totals(rows):
-    return [
-        {
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
        }
-        for row in rows
-    ]
+        yearly_totals.append(json_result)
+
+    return yearly_totals
--- a/app/billing/rest.py
+++ b/app/billing/rest.py
@@ -6,18 +6,18 @@ from app.billing.billing_schemas import (
    serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
    dao_get_free_sms_fragment_limit_for_year,
-    dao_update_annual_billing_for_future_years,
-    set_default_free_allowance_for_service,
+    dao_get_all_free_sms_fragment_limit,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
-    fetch_billing_totals_for_year,
-    fetch_monthly_billing_for_year,
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
 )
+
+from app.errors import InvalidRequest
 from app.errors import register_errors
-from app.models import Service
 from app.schema_validation import validate

 billing_blueprint = Blueprint(
@@ -30,6 +30,7 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)


+@billing_blueprint.route('/ft-monthly-usage')
@billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
    try:
@@ -41,6 +42,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
    return jsonify(data)


+@billing_blueprint.route('/ft-yearly-usage-summary')
@billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
    try:
@@ -61,14 +63,27 @@ def get_free_sms_fragment_limit(service_id):
    annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)

    if annual_billing is None:
-        service = Service.query.get(service_id)
-        # An entry does not exist in annual_billing table for that service and year.
-        # Set the annual billing to the default free allowance based on the organisation type of the service.
+        # An entry does not exist in annual_billing table for that service and year. If it is a past year,
+        # we return the oldest entry.
+        # If it is the current or future years, we create an entry in the db table using the newest record,
+        # and return that number.  If all fails, we return InvalidRequest.
+        sms_list = dao_get_all_free_sms_fragment_limit(service_id)

-        annual_billing = set_default_free_allowance_for_service(
-            service=service,
-            year_start=int(financial_year_start) if financial_year_start else None
-        )
+        if not sms_list:
+            raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
+        else:
+            if financial_year_start is None:
+                financial_year_start = get_current_financial_year_start_year()
+
+            if int(financial_year_start) < get_current_financial_year_start_year():
+                # return the earliest historical entry
+                annual_billing = sms_list[0]   # The oldest entry
+            else:
+                annual_billing = sms_list[-1]  # The newest entry
+
+                annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
+                                                                              annual_billing.free_sms_fragment_limit,
+                                                                              financial_year_start)

    return jsonify(annual_billing.serialize_free_sms_items()), 200

--- a/app/broadcast_message/broadcast_message_schema.py
+++ b/app/broadcast_message/broadcast_message_schema.py
@@ -1,60 +0,0 @@
-from app.models import BroadcastStatusType
-from app.schema_validation.definitions import uuid
-
-create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST create broadcast_message schema',
-    'type': 'object',
-    'title': 'Create broadcast_message',
-    'properties': {
-        'template_id': uuid,
-        'service_id': uuid,
-        'created_by': uuid,
-        'personalisation': {'type': 'object'},
-        'starts_at': {'type': 'string', 'format': 'datetime'},
-        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-        'content': {'type': 'string', 'minLength': 1},
-        'reference': {'type': 'string', 'minLength': 1, 'maxLength': 255},
-    },
-    'required': ['service_id', 'created_by'],
-    'allOf': [
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['content']},
-        ]},
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['reference']},
-        ]},
-    ],
-    'additionalProperties': False
-}
-
-update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST update broadcast_message schema',
-    'type': 'object',
-    'title': 'Update broadcast_message',
-    'properties': {
-        'personalisation': {'type': 'object'},
-        'starts_at': {'type': 'string', 'format': 'datetime'},
-        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-    },
-    'required': [],
-    'additionalProperties': False
-}
-
-update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'description': 'POST update broadcast_message status schema',
-    'type': 'object',
-    'title': 'Update broadcast_message',
-    'properties': {
-        'status': {'type': 'string', 'enum': BroadcastStatusType.STATUSES},
-        'created_by': uuid,
-    },
-    'required': ['status', 'created_by'],
-    'additionalProperties': False
-}
--- a/app/broadcast_message/rest.py
+++ b/app/broadcast_message/rest.py
@@ -1,165 +0,0 @@
-import iso8601
-from flask import Blueprint, jsonify, request
-from notifications_utils.template import BroadcastMessageTemplate
-
-from app.broadcast_message import utils as broadcast_utils
-from app.broadcast_message.broadcast_message_schema import (
-    create_broadcast_message_schema,
-    update_broadcast_message_schema,
-    update_broadcast_message_status_schema,
-)
-from app.dao.broadcast_message_dao import (
-    dao_get_broadcast_message_by_id_and_service_id,
-    dao_get_broadcast_messages_for_service,
-)
-from app.dao.dao_utils import dao_save_object
-from app.dao.services_dao import dao_fetch_service_by_id
-from app.dao.templates_dao import dao_get_template_by_id_and_service_id
-from app.dao.users_dao import get_user_by_id
-from app.errors import InvalidRequest, register_errors
-from app.models import BroadcastMessage, BroadcastStatusType
-from app.schema_validation import validate
-
-broadcast_message_blueprint = Blueprint(
-    'broadcast_message',
-    __name__,
-    url_prefix='/service/<uuid:service_id>/broadcast-message'
-)
-register_errors(broadcast_message_blueprint)
-
-
-def _parse_nullable_datetime(dt):
-    if dt:
-        return iso8601.parse_date(dt).replace(tzinfo=None)
-    return dt
-
-
-@broadcast_message_blueprint.route('', methods=['GET'])
-def get_broadcast_messages_for_service(service_id):
-    # TODO: should this return template content/data in some way? or can we rely on them being cached admin side.
-    # we might need stuff like template name for showing on the dashboard.
-    # TODO: should this paginate or filter on dates or anything?
-    broadcast_messages = [o.serialize() for o in dao_get_broadcast_messages_for_service(service_id)]
-    return jsonify(broadcast_messages=broadcast_messages)
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['GET'])
-def get_broadcast_message(service_id, broadcast_message_id):
-    return jsonify(dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id).serialize())
-
-
-@broadcast_message_blueprint.route('', methods=['POST'])
-def create_broadcast_message(service_id):
-    data = request.get_json()
-
-    validate(data, create_broadcast_message_schema)
-    service = dao_fetch_service_by_id(data['service_id'])
-    user = get_user_by_id(data['created_by'])
-    personalisation = data.get('personalisation', {})
-    template_id = data.get('template_id')
-
-    if template_id:
-        template = dao_get_template_by_id_and_service_id(
-            template_id, data['service_id']
-        )
-        content = str(template._as_utils_template_with_personalisation(
-            personalisation
-        ))
-        reference = None
-    else:
-        temporary_template = BroadcastMessageTemplate.from_content(data['content'])
-        if temporary_template.content_too_long:
-            raise InvalidRequest(
-                (
-                    f'Content must be '
-                    f'{temporary_template.max_content_count:,.0f} '
-                    f'characters or fewer'
-                ) + (
-                    ' (because it could not be GSM7 encoded)'
-                    if temporary_template.non_gsm_characters else ''
-                ),
-                status_code=400,
-            )
-        template = None
-        content = str(temporary_template)
-        reference = data['reference']
-
-    broadcast_message = BroadcastMessage(
-        service_id=service.id,
-        template_id=template_id,
-        template_version=template.version if template else None,
-        personalisation=personalisation,
-        areas=data.get("areas", {}),
-        status=BroadcastStatusType.DRAFT,
-        starts_at=_parse_nullable_datetime(data.get('starts_at')),
-        finishes_at=_parse_nullable_datetime(data.get('finishes_at')),
-        created_by_id=user.id,
-        content=content,
-        reference=reference,
-        stubbed=service.restricted
-    )
-
-    dao_save_object(broadcast_message)
-
-    return jsonify(broadcast_message.serialize()), 201
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['POST'])
-def update_broadcast_message(service_id, broadcast_message_id):
-    data = request.get_json()
-    validate(data, update_broadcast_message_schema)
-
-    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
-
-    if broadcast_message.status not in BroadcastStatusType.PRE_BROADCAST_STATUSES:
-        raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id} while it has status {broadcast_message.status}',
-            status_code=400
-        )
-
-    areas = data.get("areas", {})
-
-    if ('ids' in areas and 'simple_polygons' not in areas) or ('ids' not in areas and 'simple_polygons' in areas):
-        raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.',
-            status_code=400
-        )
-
-    if 'personalisation' in data:
-        broadcast_message.personalisation = data['personalisation']
-    if 'starts_at' in data:
-        broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at'])
-    if 'finishes_at' in data:
-        broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at'])
-    if 'ids' in areas and 'simple_polygons' in areas:
-        broadcast_message.areas = areas
-
-    dao_save_object(broadcast_message)
-
-    return jsonify(broadcast_message.serialize()), 200
-
-
-@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>/status', methods=['POST'])
-def update_broadcast_message_status(service_id, broadcast_message_id):
-    data = request.get_json()
-
-    validate(data, update_broadcast_message_status_schema)
-    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
-
-    if not broadcast_message.service.active:
-        raise InvalidRequest("Updating broadcast message is not allowed: service is inactive ", 403)
-
-    new_status = data['status']
-    updating_user = get_user_by_id(data['created_by'])
-
-    if updating_user not in broadcast_message.service.users:
-        #  we allow platform admins to cancel broadcasts, and we don't check user if request was done via API
-        if not (new_status == BroadcastStatusType.CANCELLED and updating_user.platform_admin):
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot update broadcast_message {broadcast_message.id} from other service',
-                status_code=400
-            )
-
-    broadcast_utils.update_broadcast_message_status(broadcast_message, new_status, updating_user)
-
-    return jsonify(broadcast_message.serialize()), 200
--- a/app/broadcast_message/translators.py
+++ b/app/broadcast_message/translators.py
@@ -1,37 +0,0 @@
-from bs4 import BeautifulSoup
-
-
-def cap_xml_to_dict(cap_xml):
-    # This function assumes that it’s being passed valid CAP XML
-    cap = BeautifulSoup(cap_xml, "xml")
-    return {
-        "msgType": cap.alert.msgType.text,
-        "reference": cap.alert.identifier.text,
-        "references": (
-            # references to previous events belonging to the same alert
-            cap.alert.references.text if cap.alert.references else None
-        ),
-        "cap_event": cap.alert.info.event.text,
-        "category": cap.alert.info.category.text,
-        "expires": cap.alert.info.expires.text,
-        "content": cap.alert.info.description.text,
-        "areas": [
-            {
-                "name": area.areaDesc.text,
-                "polygons": [
-                    cap_xml_polygon_to_list(polygon.text)
-                    for polygon in area.find_all('polygon')
-                ]
-            }
-            for area in cap.alert.info.find_all('area')
-        ]
-    }
-
-
-def cap_xml_polygon_to_list(polygon_string):
-    return [
-        [
-            float(coordinate) for coordinate in pair.split(',')
-        ]
-        for pair in polygon_string.strip().split(' ')
-    ]
--- a/app/broadcast_message/utils.py
+++ b/app/broadcast_message/utils.py
@@ -1,140 +0,0 @@
-import inspect
-from datetime import datetime
-
-from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-
-from app import zendesk_client
-from app.celery.broadcast_message_tasks import send_broadcast_event
-from app.config import QueueNames
-from app.dao.dao_utils import dao_save_object
-from app.errors import InvalidRequest
-from app.models import (
-    BroadcastEvent,
-    BroadcastEventMessageType,
-    BroadcastStatusType,
-)
-
-
-def update_broadcast_message_status(broadcast_message, new_status, updating_user=None, api_key_id=None):
-    _validate_broadcast_update(broadcast_message, new_status, updating_user)
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        broadcast_message.approved_at = datetime.utcnow()
-        broadcast_message.approved_by = updating_user
-
-    if new_status == BroadcastStatusType.CANCELLED:
-        broadcast_message.cancelled_at = datetime.utcnow()
-        broadcast_message.cancelled_by = updating_user
-        broadcast_message.cancelled_by_api_key_id = api_key_id
-
-    current_app.logger.info(
-        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
-    )
-    broadcast_message.status = new_status
-
-    dao_save_object(broadcast_message)
-    _create_p1_zendesk_alert(broadcast_message)
-
-    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
-        _create_broadcast_event(broadcast_message)
-
-
-def _validate_broadcast_update(broadcast_message, new_status, updating_user):
-    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
-        raise InvalidRequest(
-            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
-            status_code=400
-        )
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        # training mode services can approve their own broadcasts
-        if updating_user == broadcast_message.created_by and not broadcast_message.service.restricted:
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
-                status_code=400
-            )
-        elif len(broadcast_message.areas['simple_polygons']) == 0:
-            raise InvalidRequest(
-                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
-                status_code=400
-            )
-
-
-def _create_p1_zendesk_alert(broadcast_message):
-    if current_app.config['NOTIFY_ENVIRONMENT'] != 'live':
-        return
-
-    if broadcast_message.status != BroadcastStatusType.BROADCASTING:
-        return
-
-    message = inspect.cleandoc(f"""
-        Broadcast Sent
-
-        https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}
-
-        Sent on channel {broadcast_message.service.broadcast_channel} to {broadcast_message.areas["names"]}.
-
-        Content starts "{broadcast_message.content[:100]}".
-
-        Follow the runbook to check the broadcast went out OK:
-        https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs/edit#heading=h.lzr9aq5b4wg
-    """)
-
-    ticket = NotifySupportTicket(
-        subject='Live broadcast sent',
-        message=message,
-        ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-        technical_ticket=True,
-        org_id=current_app.config['BROADCAST_ORGANISATION_ID'],
-        org_type='central',
-        service_id=str(broadcast_message.service_id),
-        p1=True
-    )
-    zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-def _create_broadcast_event(broadcast_message):
-    """
-    If the service is live and the broadcast message is not stubbed, creates a broadcast event, stores it in the
-    database, and triggers the task to send the CAP XML off.
-    """
-    service = broadcast_message.service
-
-    if not broadcast_message.stubbed and not service.restricted:
-        msg_types = {
-            BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
-            BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
-        }
-
-        event = BroadcastEvent(
-            service=service,
-            broadcast_message=broadcast_message,
-            message_type=msg_types[broadcast_message.status],
-            transmitted_content={"body": broadcast_message.content},
-            transmitted_areas=broadcast_message.areas,
-            # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it
-            # include a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
-            transmitted_sender='notifications.service.gov.uk',
-
-            # TODO: Should this be set to now? Or the original starts_at?
-            transmitted_starts_at=broadcast_message.starts_at,
-            transmitted_finishes_at=broadcast_message.finishes_at,
-        )
-
-        dao_save_object(event)
-
-        send_broadcast_event.apply_async(
-            kwargs={'broadcast_event_id': str(event.id)},
-            queue=QueueNames.BROADCASTS
-        )
-    elif broadcast_message.stubbed != service.restricted:
-        # It's possible for a service to create a broadcast in trial mode, and then approve it after the
-        # service is live (or vice versa). We don't think it's safe to send such broadcasts, as the service
-        # has changed since they were created. Log an error instead.
-        current_app.logger.error(
-            f'Broadcast event not created. Stubbed status of broadcast message was {broadcast_message.stubbed}'
-            f' but service was {"in trial mode" if service.restricted else "live"}'
-        )
--- a/app/celery/broadcast_message_tasks.py
+++ b/app/celery/broadcast_message_tasks.py
@@ -1,230 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import cbc_proxy_client, notify_celery
-from app.clients.cbc_proxy import CBCProxyRetryableException
-from app.config import QueueNames, TaskNames
-from app.dao.broadcast_message_dao import (
-    create_broadcast_provider_message,
-    dao_get_broadcast_event_by_id,
-    update_broadcast_provider_message_status,
-)
-from app.models import (
-    BroadcastEventMessageType,
-    BroadcastProvider,
-    BroadcastProviderMessageStatus,
-)
-from app.utils import format_sequential_number
-
-
-class BroadcastIntegrityError(Exception):
-    pass
-
-
-def get_retry_delay(retry_count):
-    """
-    Given a count of retries so far, return a delay for the next one.
-    `retry_count` should be 0 the first time a task fails.
-    """
-    # TODO: replace with celery's built in exponential backoff
-
-    # 2 to the power of x. 1, 2, 4, 8, 16, 32, ...
-    delay = 2**retry_count
-    # never wait longer than 4 minutes
-    return min(delay, 240)
-
-
-def check_event_is_authorised_to_be_sent(broadcast_event, provider):
-    if not broadcast_event.service.active:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is suspended'
-        )
-
-    if broadcast_event.service.restricted:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is not live'
-        )
-
-    if broadcast_event.broadcast_message.stubbed:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the broadcast message is stubbed'
-        )
-
-
-def check_event_makes_sense_in_sequence(broadcast_event, provider):
-    """
-    If any previous event hasn't sent yet for that provider, then we shouldn't send the current event. Instead, fail and
-    raise a zendesk ticket - so that a notify team member can assess the state of the previous messages, and if
-    necessary, can replay the `send_broadcast_provider_message` task if the previous message has now been sent.
-
-    Note: This is called before the new broadcast_provider_message is created.
-
-    # Help, I've come across this code following a pagerduty alert, what should I do?
-
-    1. Find the failing broadcast_provider_message associated with the previous event that caused this to trip.
-    2. If that provider message is still failing to send, fix the issue causing that. The task to send that previous
-       message might still be retrying in the background - look for logs related to that task.
-    3. If that provider message has sent succesfully, you might need to send this task off depending on context. This
-       might not always be true though, for example, it may not be necessary to send a cancel if the original alert has
-       already expired.
-    4. If you need to re-send this task off again, you'll need to run the following command on paas:
-       `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)`
-    """
-    current_provider_message = broadcast_event.get_provider_message(provider)
-    # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'It is in status {current_provider_message.status}'
-        )
-
-    if broadcast_event.transmitted_finishes_at < datetime.utcnow():
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'The expiry time of {broadcast_event.transmitted_finishes_at} has already passed'
-        )
-
-    # get events sorted from earliest to latest
-    events = sorted(broadcast_event.broadcast_message.events, key=lambda x: x.sent_at)
-
-    for prev_event in events:
-        if prev_event.id != broadcast_event.id and prev_event.sent_at < broadcast_event.sent_at:
-            # get the record from when that event was sent to the same provider
-            prev_provider_message = prev_event.get_provider_message(provider)
-
-            # the previous message hasn't even got round to running `send_broadcast_provider_message` yet.
-            if not prev_provider_message:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has no provider_message for provider {provider} yet.\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-            # if there's a previous message that has started but not finished sending (whether it fatally errored or is
-            # currently retrying)
-            if prev_provider_message.status != BroadcastProviderMessageStatus.ACK:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has not finished sending to provider {provider} yet.\n' +
-                    f'It is currently in status "{prev_provider_message.status}".\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-
-@notify_celery.task(name="send-broadcast-event")
-def send_broadcast_event(broadcast_event_id):
-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
-
-    for provider in broadcast_event.service.get_available_broadcast_providers():
-        send_broadcast_provider_message.apply_async(
-            kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider},
-            queue=QueueNames.BROADCASTS
-        )
-
-
-# max_retries=None: retry forever
-@notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None)
-def send_broadcast_provider_message(self, broadcast_event_id, provider):
-    if not current_app.config['CBC_PROXY_ENABLED']:
-        current_app.logger.info(
-            "CBC Proxy disabled, not sending broadcast_provider_message for "
-            f"broadcast_event_id {broadcast_event_id} with provider {provider}"
-        )
-        return
-
-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    check_event_is_authorised_to_be_sent(broadcast_event, provider)
-    check_event_makes_sense_in_sequence(broadcast_event, provider)
-
-    # the broadcast_provider_message may already exist if we retried previously
-    broadcast_provider_message = broadcast_event.get_provider_message(provider)
-    if broadcast_provider_message is None:
-        broadcast_provider_message = create_broadcast_provider_message(broadcast_event, provider)
-
-    formatted_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        formatted_message_number = format_sequential_number(broadcast_provider_message.message_number)
-
-    current_app.logger.info(
-        f'Invoking cbc proxy to send broadcast_provider_message with ID of {broadcast_provider_message.id} '
-        f'and broadcast_event ID of {broadcast_event_id} '
-        f'msgType {broadcast_event.message_type}'
-    )
-
-    areas = [
-        {"polygon": polygon}
-        for polygon in broadcast_event.transmitted_areas["simple_polygons"]
-    ]
-
-    cbc_proxy_provider_client = cbc_proxy_client.get_proxy(provider)
-
-    try:
-        if broadcast_event.message_type == BroadcastEventMessageType.ALERT:
-            cbc_proxy_provider_client.create_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.UPDATE:
-            cbc_proxy_provider_client.update_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                # We think an alert update should always go out on the same channel that created the alert
-                # We recognise there is a small risk with this code here that if the services channel was
-                # changed between an alert being sent out and then updated, then something might go wrong
-                # but we are relying on service channels changing almost never, and not mid incident
-                # We may consider in the future, changing this such that we store the channel a broadcast was
-                # sent on on the broadcast message itself and pick the value from there instead of the service
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.CANCEL:
-            cbc_proxy_provider_client.cancel_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-            )
-    except CBCProxyRetryableException as exc:
-        delay = get_retry_delay(self.request.retries)
-        current_app.logger.exception(
-            f'Retrying send_broadcast_provider_message for broadcast event {broadcast_event_id}, '
-            f'provider message {broadcast_provider_message.id}, provider {provider} in {delay} seconds'
-        )
-
-        self.retry(
-            exc=exc,
-            countdown=delay,
-            queue=QueueNames.BROADCASTS,
-        )
-
-    update_broadcast_provider_message_status(broadcast_provider_message, status=BroadcastProviderMessageStatus.ACK)
-
-
-@notify_celery.task(name='trigger-link-test')
-def trigger_link_test(provider):
-    cbc_proxy_client.get_proxy(provider).send_link_test()
--- a/app/celery/celery.py
+++ b/app/celery/celery.py
@@ -0,0 +1,62 @@
+import time
+
+from celery import Celery, Task
+from celery.signals import worker_process_shutdown
+from flask import current_app, g, request
+from flask.ctx import has_request_context
+
+
+@worker_process_shutdown.connect
+def worker_process_shutdown(sender, signal, pid, exitcode, **kwargs):
+    current_app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
+
+
+def make_task(app):
+    class NotifyTask(Task):
+        abstract = True
+        start = None
+
+        def on_success(self, retval, task_id, args, kwargs):
+            elapsed_time = time.time() - self.start
+            app.logger.info(
+                "{task_name} took {time}".format(
+                    task_name=self.name, time="{0:.4f}".format(elapsed_time)
+                )
+            )
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            # ensure task will log exceptions to correct handlers
+            app.logger.exception('Celery task: {} failed'.format(self.name))
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def __call__(self, *args, **kwargs):
+            # ensure task has flask context to access config, logger, etc
+            with app.app_context():
+                self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
+                return super().__call__(*args, **kwargs)
+
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
+    return NotifyTask
+
+
+class NotifyCelery(Celery):
+
+    def init_app(self, app):
+        super().__init__(
+            app.import_name,
+            broker=app.config['BROKER_URL'],
+            task_cls=make_task(app),
+        )
+
+        self.conf.update(app.config)
--- a/app/celery/letters_pdf_tasks.py
+++ b/app/celery/letters_pdf_tasks.py
@@ -1,47 +1,50 @@
-from base64 import urlsafe_b64encode
+import math
+import base64
 from datetime import datetime, timedelta
+from uuid import UUID
 from hashlib import sha512
+from base64 import urlsafe_b64encode

 from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
+from requests import (
+    post as requests_post,
+    RequestException
+)
+from celery.exceptions import MaxRetriesExceededError
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.s3 import s3upload
 from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
-from notifications_utils.postal_address import PostalAddress
 from notifications_utils.timezones import convert_utc_to_bst

 from app import encryption, notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
-from app.cronitor import cronitor
 from app.dao.notifications_dao import (
-    dao_get_letters_and_sheets_volume_by_postage,
-    dao_get_letters_to_be_printed,
-    dao_get_notification_by_reference,
-    dao_update_notification,
-    dao_update_notifications_by_reference,
    get_notification_by_id,
    update_notification_status_by_id,
+    dao_update_notification,
+    dao_get_notification_by_reference,
+    dao_update_notifications_by_reference,
+    dao_get_letters_to_be_printed,
 )
-from app.dao.templates_dao import dao_get_template_by_id
+from app.letters.utils import get_letter_pdf_filename
 from app.errors import VirusScanError
 from app.exceptions import NotificationTechnicalFailureException
 from app.letters.utils import (
-    LetterPDFNotFound,
-    ScanErrorType,
-    find_letter_pdf_in_s3,
-    generate_letter_pdf_filename,
+    copy_redaction_failed_pdf,
    get_billable_units_for_letter_page_count,
-    get_file_names_from_error_bucket,
-    get_folder_name,
    get_reference_from_filename,
-    move_error_pdf_to_scan_bucket,
+    get_folder_name,
+    upload_letter_pdf,
+    ScanErrorType,
    move_failed_pdf,
    move_sanitised_letter_to_test_or_live_pdf_bucket,
    move_scan_to_invalid_pdf_bucket,
+    move_error_pdf_to_scan_bucket,
+    get_file_names_from_error_bucket,
 )
 from app.models import (
-    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_POSTAGE_TYPES,
-    KEY_TYPE_NORMAL,
    KEY_TYPE_TEST,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
@@ -49,85 +52,75 @@ from app.models import (
    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_VALIDATION_FAILED,
    NOTIFICATION_VIRUS_SCAN_FAILED,
-    POSTAGE_TYPES,
-    RESOLVE_POSTAGE_FOR_FILE_NAME,
-    Service,
 )
+from app.cronitor import cronitor


-@notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300)
-def get_pdf_for_templated_letter(self, notification_id):
+@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def create_letters_pdf(self, notification_id):
    try:
        notification = get_notification_by_id(notification_id, _raise=True)
-        letter_filename = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=notification.key_type == KEY_TYPE_TEST,
-            postage=notification.postage
+        pdf_data, billable_units = get_letters_pdf(
+            notification.template,
+            contact_block=notification.reply_to_text,
+            filename=notification.service.letter_branding and notification.service.letter_branding.filename,
+            values=notification.personalisation
        )
-        letter_data = {
-            'letter_contact_block': notification.reply_to_text,
-            'template': {
-                "subject": notification.template.subject,
-                "content": notification.template.content,
-                "template_type": notification.template.template_type
-            },
-            'values': notification.personalisation,
-            'logo_filename': notification.service.letter_branding and notification.service.letter_branding.filename,
-            'letter_filename': letter_filename,
-            "notification_id": str(notification_id),
-            'key_type': notification.key_type
-        }

-        encrypted_data = encryption.encrypt(letter_data)
+        upload_letter_pdf(notification, pdf_data)

-        notify_celery.send_task(
-            name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER,
-            args=(encrypted_data,),
-            queue=QueueNames.SANITISE_LETTERS
-        )
-    except Exception as e:
-        try:
-            current_app.logger.exception(
-                f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
-            )
-            self.retry(exc=e, queue=QueueNames.RETRY)
-        except self.MaxRetriesExceededError:
-            message = f"RETRY FAILED: Max retries reached. " \
-                      f"The task create-letter-pdf failed for notification id {notification_id}. " \
-                      f"Notification has been updated to technical-failure"
-            update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
-            raise NotificationTechnicalFailureException(message)
-
-
-@notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300)
-def update_billable_units_for_letter(self, notification_id, page_count):
-    notification = get_notification_by_id(notification_id, _raise=True)
-
-    billable_units = get_billable_units_for_letter_page_count(page_count)
-
-    if notification.key_type != KEY_TYPE_TEST:
-        notification.billable_units = billable_units
-        dao_update_notification(notification)
+        if notification.key_type != KEY_TYPE_TEST:
+            notification.billable_units = billable_units
+            dao_update_notification(notification)

        current_app.logger.info(
-            f"Letter notification id: {notification_id} reference {notification.reference}: "
-            f"billable units set to {billable_units}"
-        )
+            'Letter notification reference {reference}: billable units set to {billable_units}'.format(
+                reference=str(notification.reference), billable_units=billable_units))
+
+    except (RequestException, BotoClientError):
+        try:
+            current_app.logger.exception(
+                "Letters PDF notification creation for id: {} failed".format(notification_id)
+            )
+            self.retry(queue=QueueNames.RETRY)
+        except MaxRetriesExceededError:
+            current_app.logger.error(
+                "RETRY FAILED: task create_letters_pdf failed for notification {}".format(notification_id),
+            )
+            update_notification_status_by_id(notification_id, 'technical-failure')


-@notify_celery.task(
-    bind=True, name="update-validation-failed-for-templated-letter", max_retries=15, default_retry_delay=300
-)
-def update_validation_failed_for_templated_letter(self, notification_id, page_count):
-    notification = get_notification_by_id(notification_id, _raise=True)
-    notification.status = NOTIFICATION_VALIDATION_FAILED
-    dao_update_notification(notification)
-    current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}")
+def get_letters_pdf(template, contact_block, filename, values):
+    template_for_letter_print = {
+        "subject": template.subject,
+        "content": template.content
+    }
+
+    data = {
+        'letter_contact_block': contact_block,
+        'template': template_for_letter_print,
+        'values': values,
+        'filename': filename,
+    }
+    resp = requests_post(
+        '{}/print.pdf'.format(
+            current_app.config['TEMPLATE_PREVIEW_API_HOST']
+        ),
+        json=data,
+        headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+    )
+    resp.raise_for_status()
+
+    pages_per_sheet = 2
+    billable_units = math.ceil(int(resp.headers.get("X-pdf-page-count", 0)) / pages_per_sheet)
+
+    return resp.content, billable_units


@notify_celery.task(name='collate-letter-pdfs-to-be-sent')
@cronitor("collate-letter-pdfs-to-be-sent")
+@statsd(namespace="tasks")
 def collate_letter_pdfs_to_be_sent():
    """
    Finds all letters which are still waiting to be sent to DVLA for printing
@@ -143,117 +136,54 @@ def collate_letter_pdfs_to_be_sent():
    print_run_deadline = print_run_date.replace(
        hour=17, minute=30, second=0, microsecond=0
    )
-    _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)

-    for postage in POSTAGE_TYPES:
-        current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
-        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
+    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)

-        for i, letters in enumerate(group_letters(letters_to_print)):
-            filenames = [letter['Key'] for letter in letters]
+    for i, letters in enumerate(group_letters(letters_to_print)):
+        filenames = [letter['Key'] for letter in letters]

-            service_id = letters[0]['ServiceId']
-            organisation_id = letters[0]['OrganisationId']
-
-            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format(
-                date=print_run_deadline.strftime("%Y-%m-%d"),
-                postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
-                num=i + 1,
-                hash=hash,
-                service_id=service_id,
-                organisation_id=organisation_id
-            )
-
-            current_app.logger.info(
-                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                    len(filenames),
-                    dvla_filename,
-                    sum(letter['Size'] for letter in letters)
-                )
-            )
-            notify_celery.send_task(
-                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-                kwargs={
-                    'filenames_to_zip': filenames,
-                    'upload_filename': dvla_filename
-                },
-                queue=QueueNames.PROCESS_FTP,
-                compression='zlib'
-            )
-        current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}")
-
-    current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
-
-
-def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
-    letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
-    send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date())
-
-
-def send_letters_volume_email_to_dvla(letters_volumes, date):
-    personalisation = {
-        'total_volume': 0,
-        'first_class_volume': 0,
-        'second_class_volume': 0,
-        'international_volume': 0,
-        'total_sheets': 0,
-        'first_class_sheets': 0,
-        "second_class_sheets": 0,
-        'international_sheets': 0,
-        'date': date.strftime("%d %B %Y")
-    }
-    for item in letters_volumes:
-        personalisation['total_volume'] += item.letters_count
-        personalisation['total_sheets'] += item.sheets_count
-        if f"{item.postage}_class_volume" in personalisation:
-            personalisation[f"{item.postage}_class_volume"] = item.letters_count
-            personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
-        else:
-            personalisation["international_volume"] += item.letters_count
-            personalisation["international_sheets"] += item.sheets_count
-
-    template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
-    recipients = current_app.config['DVLA_EMAIL_ADDRESSES']
-    reply_to = template.service.get_default_reply_to_email_address()
-    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
-
-    # avoid circular imports:
-    from app.notifications.process_notifications import (
-        persist_notification,
-        send_notification_to_queue,
-    )
-    for recipient in recipients:
-        saved_notification = persist_notification(
-            template_id=template.id,
-            template_version=template.version,
-            recipient=recipient,
-            service=service,
-            personalisation=personalisation,
-            notification_type=template.template_type,
-            api_key_id=None,
-            key_type=KEY_TYPE_NORMAL,
-            reply_to_text=reply_to
+        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+            date=print_run_deadline.strftime("%Y-%m-%d"),
+            num=i + 1,
+            hash=hash
        )

-        send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+        current_app.logger.info(
+            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                len(filenames),
+                dvla_filename,
+                sum(letter['Size'] for letter in letters)
+            )
+        )
+        notify_celery.send_task(
+            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+            kwargs={
+                'filenames_to_zip': filenames,
+                'upload_filename': dvla_filename
+            },
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
+        )


-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
+
+    letter_pdfs = []
    for letter in letters_awaiting_sending:
-        try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
-            yield {
-                "Key": letter_pdf.key,
-                "Size": letter_pdf.size,
-                "ServiceId": str(letter.service_id),
-                "OrganisationId": str(letter.service.organisation_id)
-            }
-        except (BotoClientError, LetterPDFNotFound) as e:
-            current_app.logger.exception(
-                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
+        letter_file_name = get_letter_pdf_filename(
+            reference=letter.reference,
+            crown=letter.service.crown,
+            sending_date=letter.created_at,
+            postage=letter.postage
+        )
+
+        letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
+        letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
+
+    return letter_pdfs


 def group_letters(letter_pdfs):
@@ -264,23 +194,16 @@ def group_letters(letter_pdfs):
    """
    running_filesize = 0
    list_of_files = []
-    service_id = None
    for letter in letter_pdfs:
        if letter['Key'].lower().endswith('.pdf'):
-            if not service_id:
-                service_id = letter['ServiceId']
            if (
-                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']
-                or len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
-                or letter['ServiceId'] != service_id
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
            ):
                yield list_of_files
                running_filesize = 0
                list_of_files = []
-                service_id = None

-            if not service_id:
-                service_id = letter['ServiceId']
            running_filesize += letter['Size']
            list_of_files.append(letter)

@@ -288,6 +211,72 @@ def group_letters(letter_pdfs):
        yield list_of_files


+@notify_celery.task(bind=True, name='process-virus-scan-passed', max_retries=15, default_retry_delay=300)
+def process_virus_scan_passed(self, filename):
+    reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
+    current_app.logger.info('notification id {} Virus scan passed: {}'.format(notification.id, filename))
+
+    is_test_key = notification.key_type == KEY_TYPE_TEST
+
+    scan_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
+    old_pdf = scan_pdf_object.get()['Body'].read()
+
+    sanitise_response, result = _sanitise_precompiled_pdf(self, notification, old_pdf)
+    new_pdf = None
+    if result == 'validation_passed':
+        new_pdf = base64.b64decode(sanitise_response["file"].encode())
+
+        redaction_failed_message = sanitise_response.get("redaction_failed_message")
+        if redaction_failed_message and not is_test_key:
+            current_app.logger.info('{} for notification id {} ({})'.format(
+                redaction_failed_message, notification.id, filename)
+            )
+            copy_redaction_failed_pdf(filename)
+
+    billable_units = get_billable_units_for_letter_page_count(sanitise_response.get("page_count"))
+
+    # TODO: Remove this once CYSP update their template to not cross over the margins
+    if notification.service_id == UUID('fe44178f-3b45-4625-9f85-2264a36dd9ec'):  # CYSP
+        # Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
+        new_pdf = old_pdf
+
+    if result == 'validation_failed' and not new_pdf:
+        current_app.logger.info('Invalid precompiled pdf received {} ({})'.format(notification.id, filename))
+        _move_invalid_letter_and_update_status(
+            notification=notification,
+            filename=filename,
+            scan_pdf_object=scan_pdf_object,
+            message=sanitise_response["message"],
+            invalid_pages=sanitise_response.get("invalid_pages"),
+            page_count=sanitise_response.get("page_count")
+        )
+        return
+
+    current_app.logger.info('notification id {} ({}) sanitised and ready to send'.format(notification.id, filename))
+
+    try:
+        _upload_pdf_to_test_or_live_pdf_bucket(
+            new_pdf,
+            filename,
+            is_test_letter=is_test_key,
+            created_at=notification.created_at
+        )
+
+        update_letter_pdf_status(
+            reference=reference,
+            status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
+            billable_units=billable_units,
+            recipient_address=sanitise_response.get("recipient_address")
+        )
+        scan_pdf_object.delete()
+    except BotoClientError:
+        current_app.logger.exception(
+            "Error uploading letter to live pdf bucket for notification: {}".format(notification.id)
+        )
+        update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
+
+
@notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300)
 def sanitise_letter(self, filename):
    try:
@@ -297,7 +286,7 @@ def sanitise_letter(self, filename):
        current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))

        if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
-            current_app.logger.info('Sanitise letter called for notification {} which is in {} state'.format(
+            current_app.logger.info('Sanitise letter called for notification {} which has is in {} state'.format(
                notification.id, notification.status))
            return

@@ -306,9 +295,6 @@ def sanitise_letter(self, filename):
            kwargs={
                'notification_id': str(notification.id),
                'filename': filename,
-                'allow_international_letters': notification.service.has_permission(
-                    INTERNATIONAL_LETTERS
-                ),
            },
            queue=QueueNames.SANITISE_LETTERS,
        )
@@ -326,8 +312,8 @@ def sanitise_letter(self, filename):
            raise NotificationTechnicalFailureException(message)


-@notify_celery.task(bind=True, name='process-sanitised-letter', max_retries=15, default_retry_delay=300)
-def process_sanitised_letter(self, sanitise_data):
+@notify_celery.task(name='process-sanitised-letter')
+def process_sanitised_letter(sanitise_data):
    letter_details = encryption.decrypt(sanitise_data)

    filename = letter_details['filename']
@@ -338,7 +324,7 @@ def process_sanitised_letter(self, sanitise_data):

    if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
        current_app.logger.info(
-            'process-sanitised-letter task called for notification {} which is in {} state'.format(
+            'process-sanitised-letter task called for notification {} which has is in {} state'.format(
                notification.id, notification.status)
        )
        return
@@ -366,8 +352,9 @@ def process_sanitised_letter(self, sanitise_data):
        billable_units = get_billable_units_for_letter_page_count(letter_details['page_count'])
        is_test_key = notification.key_type == KEY_TYPE_TEST

-        # Updating the notification needs to happen before the file is moved. This is so that if updating the
-        # notification fails, the task can retry because the file is in the same place.
+        move_sanitised_letter_to_test_or_live_pdf_bucket(filename, is_test_key, notification.created_at)
+        # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
+        original_pdf_object.delete()
        update_letter_pdf_status(
            reference=notification.reference,
            status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
@@ -375,44 +362,12 @@ def process_sanitised_letter(self, sanitise_data):
            recipient_address=letter_details['address']
        )

-        # The original filename could be wrong because we didn't know the postage.
-        # Now we know if the letter is international, we can check what the filename should be.
-        upload_file_name = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-        move_sanitised_letter_to_test_or_live_pdf_bucket(
-            filename,
-            is_test_key,
-            notification.created_at,
-            upload_file_name,
-        )
-        # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
-        original_pdf_object.delete()
-
    except BotoClientError:
-        # Boto exceptions are likely to be caused by the file(s) being in the wrong place, so retrying won't help -
-        # we'll need to manually investigate
        current_app.logger.exception(
-            f"Boto error when processing sanitised letter for notification {notification.id} (file {filename})"
+            "Boto error when processing sanitised letter for notification {}".format(filename, notification.id)
        )
        update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
        raise NotificationTechnicalFailureException
-    except Exception:
-        try:
-            current_app.logger.exception(
-                "RETRY: calling process_sanitised_letter task for notification {} failed".format(notification.id)
-            )
-            self.retry(queue=QueueNames.RETRY)
-        except self.MaxRetriesExceededError:
-            message = "RETRY FAILED: Max retries reached. " \
-                      "The task process_sanitised_letter failed for notification {}. " \
-                      "Notification has been updated to technical-failure".format(notification.id)
-            update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
-            raise NotificationTechnicalFailureException(message)


 def _move_invalid_letter_and_update_status(
@@ -439,6 +394,60 @@ def _move_invalid_letter_and_update_status(
        raise NotificationTechnicalFailureException


+def _upload_pdf_to_test_or_live_pdf_bucket(pdf_data, filename, is_test_letter, created_at):
+    target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME'
+    target_bucket_name = current_app.config[target_bucket_config]
+    target_filename = get_folder_name(created_at, dont_use_sending_date=is_test_letter) + filename
+
+    s3upload(
+        filedata=pdf_data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=target_bucket_name,
+        file_location=target_filename
+    )
+
+
+def _sanitise_precompiled_pdf(self, notification, precompiled_pdf):
+    try:
+        response = requests_post(
+            '{}/precompiled/sanitise'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST']
+            ),
+            data=precompiled_pdf,
+            headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY']),
+                     'Service-ID': str(notification.service_id),
+                     'Notification-ID': str(notification.id)}
+        )
+        response.raise_for_status()
+        return response.json(), "validation_passed"
+    except RequestException as ex:
+        if ex.response is not None and ex.response.status_code == 400:
+            message = "sanitise_precompiled_pdf validation error for notification: {}. ".format(notification.id)
+            if response.json().get("message"):
+                message += response.json()["message"]
+                if response.json().get("invalid_pages"):
+                    message += (" on pages: " + ", ".join(map(str, response.json()["invalid_pages"])))
+
+            current_app.logger.info(
+                message
+            )
+            return response.json(), "validation_failed"
+
+        try:
+            current_app.logger.exception(
+                "sanitise_precompiled_pdf failed for notification: {}".format(notification.id)
+            )
+            self.retry(queue=QueueNames.RETRY)
+        except MaxRetriesExceededError:
+            current_app.logger.error(
+                "RETRY FAILED: sanitise_precompiled_pdf failed for notification {}".format(notification.id),
+            )
+
+            notification.status = NOTIFICATION_TECHNICAL_FAILURE
+            dao_update_notification(notification)
+            raise
+
+
@notify_celery.task(name='process-virus-scan-failed')
 def process_virus_scan_failed(filename):
    move_failed_pdf(filename, ScanErrorType.FAILURE)
@@ -477,19 +486,10 @@ def process_virus_scan_error(filename):


 def update_letter_pdf_status(reference, status, billable_units, recipient_address=None):
-    postage = None
-    if recipient_address:
-        # fix allow_international_letters
-        postage = PostalAddress(raw_address=recipient_address.replace(',', '\n'),
-                                allow_international_letters=True
-                                ).postage
-        postage = postage if postage in INTERNATIONAL_POSTAGE_TYPES else None
+
    update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()}
-    if postage:
-        update_dict.update({'postage': postage, 'international': True})
    if recipient_address:
        update_dict['to'] = recipient_address
-        update_dict['normalised_to'] = ''.join(recipient_address.split()).lower()
    return dao_update_notifications_by_reference(
        references=[reference],
        update_dict=update_dict)[0]
@@ -511,9 +511,9 @@ def replay_letters_in_error(filename=None):
            )
        else:
            # stub out antivirus in dev
-            sanitise_letter.apply_async(
-                [filename],
-                queue=QueueNames.LETTERS
+            process_virus_scan_passed.apply_async(
+                kwargs={'filename': filename},
+                queue=QueueNames.LETTERS,
            )
    else:
        error_files = get_file_names_from_error_bucket()
@@ -530,41 +530,7 @@ def replay_letters_in_error(filename=None):
                )
            else:
                # stub out antivirus in dev
-                sanitise_letter.apply_async(
-                    [filename],
-                    queue=QueueNames.LETTERS
+                process_virus_scan_passed.apply_async(
+                    kwargs={'filename': moved_file_name},
+                    queue=QueueNames.LETTERS,
                )
-
-
-@notify_celery.task(name='resanitise-pdf')
-def resanitise_pdf(notification_id):
-    """
-    `notification_id` is the notification id for a PDF letter which was either uploaded or sent using the API.
-
-    This task calls the `recreate_pdf_for_precompiled_letter` template preview task which recreates the
-    PDF for a letter which is already sanitised and in the letters-pdf bucket. The new file that is generated
-    will then overwrite the existing letter in the letters-pdf bucket.
-    """
-    notification = get_notification_by_id(notification_id)
-
-    # folder_name is the folder that the letter is in the letters-pdf bucket e.g. '2021-10-10/'
-    folder_name = get_folder_name(notification.created_at)
-
-    filename = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-    notify_celery.send_task(
-        name=TaskNames.RECREATE_PDF_FOR_PRECOMPILED_LETTER,
-        kwargs={
-            'notification_id': str(notification.id),
-            'file_location': f'{folder_name}{filename}',
-            'allow_international_letters': notification.service.has_permission(
-                INTERNATIONAL_LETTERS
-            ),
-        },
-        queue=QueueNames.SANITISE_LETTERS,
-    )
--- a/app/celery/nightly_tasks.py
+++ b/app/celery/nightly_tasks.py
@@ -1,56 +1,55 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)

 import pytz
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from notifications_utils.timezones import convert_utc_to_bst
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError

-from app import notify_celery, statsd_client, zendesk_client
+from app import notify_celery, performance_platform_client, zendesk_client
 from app.aws import s3
+from app.celery.service_callback_tasks import (
+    send_delivery_status_to_service,
+    create_delivery_status_callback_data,
+)
 from app.config import QueueNames
-from app.cronitor import cronitor
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
-    dao_archive_job,
    dao_get_jobs_older_than_data_retention,
+    dao_archive_job
 )
 from app.dao.notifications_dao import (
-    dao_get_notifications_processing_time_stats,
    dao_timeout_notifications,
-    get_service_ids_with_notifications_before,
-    move_notifications_to_notification_history,
-)
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_for_all_services_by_notification_type,
+    delete_notifications_older_than_retention_by_type,
 )
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_SENDING,
-    SMS_TYPE,
-    FactProcessingTime,
    Notification,
+    NOTIFICATION_SENDING,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    KEY_TYPE_NORMAL
 )
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
+from app.performance_platform import total_sent_notifications, processing_time
+from app.cronitor import cronitor
 from app.utils import get_london_midnight_in_utc


@notify_celery.task(name="remove_sms_email_jobs")
@cronitor("remove_sms_email_jobs")
+@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
    _remove_csv_files([EMAIL_TYPE, SMS_TYPE])


@notify_celery.task(name="remove_letter_jobs")
@cronitor("remove_letter_jobs")
+@statsd(namespace="tasks")
 def remove_letter_csv_files():
    _remove_csv_files([LETTER_TYPE])

@@ -63,112 +62,142 @@ def _remove_csv_files(job_types):
        current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))


-@notify_celery.task(name="delete-notifications-older-than-retention")
-def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-
-
@notify_celery.task(name="delete-sms-notifications")
@cronitor("delete-sms-notifications")
+@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('sms')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('sms')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise


@notify_celery.task(name="delete-email-notifications")
@cronitor("delete-email-notifications")
+@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('email')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('email')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete email notifications")
+        raise


@notify_celery.task(name="delete-letter-notifications")
@cronitor("delete-letter-notifications")
+@statsd(namespace="tasks")
 def delete_letter_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('letter')
-
-
-def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
-
-    for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_london_midnight_in_utc(
-            convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
-        )
-
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': f.service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': day_to_delete_backwards_from
-        })
-
-    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7))
-    service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
-
-    # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
-    # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
-        notification_type,
-        seven_days_ago
-    )
-
-    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
-
-    for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': seven_days_ago
-        })
-
-    current_app.logger.info(
-        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
-        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
-        f'{len(service_ids_to_purge)} services without flexible data retention'
-    )
-
-
-@notify_celery.task(name='delete-notifications-for-service-and-type')
-def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
-    start = datetime.utcnow()
-    num_deleted = move_notifications_to_notification_history(
-        notification_type,
-        service_id,
-        datetime_to_delete_before,
-    )
-    if num_deleted:
-        end = datetime.utcnow()
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('letter')
        current_app.logger.info(
-            f'delete-notifications-for-service-and-type: '
-            f'service: {service_id}, '
-            f'notification_type: {notification_type}, '
-            f'count deleted: {num_deleted}, '
-            f'duration: {(end - start).seconds} seconds'
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
+                'letter',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete letter notifications")
+        raise


@notify_celery.task(name='timeout-sending-notifications')
@cronitor('timeout-sending-notifications')
+@statsd(namespace="tasks")
 def timeout_notifications():
-    notifications = ['dummy value so len() > 0']
+    technical_failure_notifications, temporary_failure_notifications = \
+        dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))

-    cutoff_time = datetime.utcnow() - timedelta(
-        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
+    notifications = technical_failure_notifications + temporary_failure_notifications
+    for notification in notifications:
+        # queue callback task only if the service_callback_api exists
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
+
+    current_app.logger.info(
+        "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    if technical_failure_notifications:
+        message = "{} notifications have been updated to technical-failure because they " \
+                  "have timed out and are still in created.Notification ids: {}".format(
+                      len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
+        raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@cronitor('send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_platform_stats(date=None):
+    # date is a string in the format of "YYYY-MM-DD"
+    if date is None:
+        date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        date = datetime.strptime(date, "%Y-%m-%d").date()
+
+    if performance_platform_client.active:
+
+        send_total_sent_notifications_to_performance_platform(bst_date=date)
+        processing_time.send_processing_time_to_performance_platform(bst_date=date)
+
+
+def send_total_sent_notifications_to_performance_platform(bst_date):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
+    start_time = get_london_midnight_in_utc(bst_date)
+
+    email_sent_count = count_dict['email']
+    sms_sent_count = count_dict['sms']
+    letter_sent_count = count_dict['letter']
+
+    current_app.logger.info(
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
    )

-    while len(notifications) > 0:
-        notifications = dao_timeout_notifications(cutoff_time)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'sms',
+        sms_sent_count
+    )

-        for notification in notifications:
-            statsd_client.incr(f'timeout-sending.{notification.sent_by}')
-            check_and_queue_callback_task(notification)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'email',
+        email_sent_count
+    )

-        current_app.logger.info(
-            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'letter',
+        letter_sent_count
+    )


@notify_celery.task(name="delete-inbound-sms")
@cronitor("delete-inbound-sms")
+@statsd(namespace="tasks")
 def delete_inbound_sms():
    try:
        start = datetime.utcnow()
@@ -185,59 +214,81 @@ def delete_inbound_sms():
        raise


+# TODO: remove me, i'm not being run by anything
+@notify_celery.task(name="delete_dvla_response_files")
+@statsd(namespace="tasks")
+def delete_dvla_response_files_older_than_seven_days():
+    try:
+        start = datetime.utcnow()
+        bucket_objects = s3.get_s3_bucket_objects(
+            current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+            'root/dispatch'
+        )
+        older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects)
+
+        for f in older_than_seven_days:
+            s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key'])
+
+        current_app.logger.info(
+            "Delete dvla response files started {} finished {} deleted {} files".format(
+                start,
+                datetime.utcnow(),
+                len(older_than_seven_days)
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete dvla response files")
+        raise
+
+
@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
@cronitor("raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
-    still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
-
-    if still_sending_count:
-        message = "There are {} letters in the 'sending' state from {}".format(
-            still_sending_count,
-            sent_date.strftime('%A %d %B')
-        )
-        # Only send alerts in production
-        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"  # noqa
-
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
-                email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
-                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
-            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-        else:
-            current_app.logger.info(message)
-
-
-def get_letter_notifications_still_sending_when_they_shouldnt_be():
    today = datetime.utcnow().date()

    # Do nothing on the weekend
    if today.isoweekday() in {6, 7}:  # sat, sun
-        return 0, None
+        return

    if today.isoweekday() in {1, 2}:  # mon, tues. look for files from before the weekend
        offset_days = 4
    else:
        offset_days = 2

-    expected_sent_date = today - timedelta(days=offset_days)
-
    q = Notification.query.filter(
        Notification.notification_type == LETTER_TYPE,
        Notification.status == NOTIFICATION_SENDING,
        Notification.key_type == KEY_TYPE_NORMAL,
-        func.date(Notification.sent_at) <= expected_sent_date
+        func.date(Notification.sent_at) <= today - timedelta(days=offset_days)
    )

-    return q.count(), expected_sent_date
+    if today.isoweekday() in {2, 4}:  # on tue, thu, we only care about first class letters
+        q = q.filter(
+            Notification.postage == 'first'
+        )
+
+    still_sending = q.count()
+
+    if still_sending:
+        message = "There are {} letters in the 'sending' state from {}".format(
+            still_sending,
+            (today - timedelta(days=offset_days)).strftime('%A %d %B')
+        )
+        # Only send alerts in production
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=message,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+        else:
+            current_app.logger.info(message)


@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
@cronitor('raise-alert-if-no-letter-ack-file')
+@statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
    # get a list of zip files since yesterday
    zip_file_set = set()
@@ -255,57 +306,33 @@ def letter_raise_alert_if_no_ack_file_for_zip():

    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                              subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
-
-    message = '\n'.join([
-        "Letter ack file does not contain all zip files sent."
-        "",
-        f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n",  # noqa
-        f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent",  # noqa
-        f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
-        "",
-        f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
-    ])
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))

+    message = (
+        "Letter ack file does not contain all zip files sent. "
+        "Missing ack for zip files: {}, "
+        "pdf bucket: {}, subfolder: {}, "
+        "ack bucket: {}"
+    ).format(
+        str(sorted(zip_file_set - ack_file_set)),
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
+        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
+    )
    # strip empty element before comparison
    ack_file_set.discard('')
    zip_file_set.discard('')

    if len(zip_file_set - ack_file_set) > 0:
        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
                subject="Letter acknowledge error",
                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
        current_app.logger.error(message)

    if len(ack_file_set - zip_file_set) > 0:
        current_app.logger.info(
            "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
        )
-
-
-@notify_celery.task(name='save-daily-notification-processing-time')
-@cronitor("save-daily-notification-processing-time")
-def save_daily_notification_processing_time(bst_date=None):
-    # bst_date is a string in the format of "YYYY-MM-DD"
-    if bst_date is None:
-        # if a date is not provided, we run against yesterdays data
-        bst_date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        bst_date = datetime.strptime(bst_date, "%Y-%m-%d").date()
-
-    start_time = get_london_midnight_in_utc(bst_date)
-    end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
-    result = dao_get_notifications_processing_time_stats(start_time, end_time)
-    insert_update_processing_time(
-        FactProcessingTime(
-            bst_date=bst_date,
-            messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs
-        )
-    )
--- a/app/celery/process_ses_receipts_tasks.py
+++ b/app/celery/process_ses_receipts_tasks.py
@@ -3,30 +3,32 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound

 from app import notify_celery, statsd_client
-from app.clients.email.aws_ses import get_aws_responses
 from app.config import QueueNames
+from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
+
 from app.notifications.notifications_ses_callback import (
-    _check_and_queue_complaint_callback_task,
-    check_and_queue_callback_task,
    determine_notification_bounce_type,
    handle_complaint,
+    _check_and_queue_complaint_callback_task,
+    _check_and_queue_callback_task,
 )


@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_ses_results(self, response):
    try:
        ses_message = json.loads(response['Message'])
        notification_type = ses_message['notificationType']
-        bounce_message = None

        if notification_type == 'Bounce':
-            notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
        elif notification_type == 'Complaint':
            _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
            return True
@@ -37,46 +39,40 @@ def process_ses_results(self, response):
        reference = ses_message['mail']['messageId']

        try:
-            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
+            notification = notifications_dao.dao_get_notification_by_reference(reference)
        except NoResultFound:
            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
            if datetime.utcnow() - message_time < timedelta(minutes=5):
-                current_app.logger.info(
-                    f"notification not found for reference: {reference} (update to {notification_status}). "
-                    f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
-                )
                self.retry(queue=QueueNames.RETRY)
            else:
                current_app.logger.warning(
-                    f"notification not found for reference: {reference} (update to {notification_status})"
+                    "notification not found for reference: {} (update to {})".format(reference, notification_status)
                )
            return

-        if bounce_message:
-            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
-
-        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
-            notifications_dao._duplicate_update_warning(
-                notification=notification,
-                status=notification_status
-            )
+        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
+            notifications_dao._duplicate_update_warning(notification, notification_status)
            return
-        else:
-            notifications_dao.dao_update_notifications_by_reference(
-                references=[reference],
-                update_dict={'status': notification_status}
+
+        notifications_dao._update_notification_status(notification=notification, status=notification_status)
+
+        if not aws_response_dict['success']:
+            current_app.logger.info(
+                "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
+                    notification.id, reference, aws_response_dict['message']
+                )
            )
+        else:
+            current_app.logger.info('SES callback return status of {} for notification: {}'.format(
+                notification_status, notification.id
+            ))

        statsd_client.incr('callback.ses.{}'.format(notification_status))

        if notification.sent_at:
-            statsd_client.timing_with_dates(
-                f'callback.ses.{notification_status}.elapsed-time',
-                datetime.utcnow(),
-                notification.sent_at
-            )
+            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)

-        check_and_queue_callback_task(notification)
+        _check_and_queue_callback_task(notification)

        return True

--- a/app/celery/process_sms_client_response_tasks.py
+++ b/app/celery/process_sms_client_response_tasks.py
@@ -1,92 +0,0 @@
-import uuid
-from datetime import datetime
-
-import pytest
-from flask import current_app
-from notifications_utils.template import SMSMessageTemplate
-
-from app import notify_celery, statsd_client
-from app.clients import ClientException
-from app.dao import notifications_dao
-from app.dao.templates_dao import dao_get_template_by_id
-from app.models import NOTIFICATION_PENDING
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
-
-# sms_response_mapper = {
-#     'MMG': get_mmg_responses,
-#     'Firetext': get_firetext_responses,
-# }
-
-
-gUpdate with new providers")
-@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
-def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
-    # validate reference
-    try:
-        uuid.UUID(provider_reference, version=4)
-    except ValueError as e:
-        current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
-        raise e
-
-    response_parser = sms_response_mapper[client_name]
-
-    # validate status
-    try:
-        notification_status, detailed_status = response_parser(status, detailed_status_code)
-        current_app.logger.info(
-            f'{client_name} callback returned status of {notification_status}'
-            f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
-        )
-    except KeyError:
-        _process_for_status(
-            notification_status='technical-failure',
-            client_name=client_name,
-            provider_reference=provider_reference
-        )
-        raise ClientException(f'{client_name} callback failed: status {status} not found.')
-
-    _process_for_status(
-        notification_status=notification_status,
-        client_name=client_name,
-        provider_reference=provider_reference,
-        detailed_status_code=detailed_status_code
-    )
-
-
-def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):
-    # record stats
-    notification = notifications_dao.update_notification_status_by_id(
-        notification_id=provider_reference,
-        status=notification_status,
-        sent_by=client_name.lower(),
-        detailed_status_code=detailed_status_code
-    )
-    if not notification:
-        return
-
-    statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
-
-    if notification.sent_at:
-        statsd_client.timing_with_dates(
-            f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
-            datetime.utcnow(),
-            notification.sent_at
-        )
-
-    if notification.billable_units == 0:
-        service = notification.service
-        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
-
-        template = SMSMessageTemplate(
-            template_model.__dict__,
-            values=notification.personalisation,
-            prefix=service.name,
-            show_prefix=service.prefix_sms,
-        )
-        notification.billable_units = template.fragment_count
-        notifications_dao.dao_update_notification(notification)
-
-    if notification_status != NOTIFICATION_PENDING:
-        check_and_queue_callback_task(notification)
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -1,10 +1,9 @@
 from flask import current_app
+from notifications_utils.recipients import InvalidEmailError
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound

 from app import notify_celery
-from app.clients.email import EmailClientNonRetryableException
-from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
-from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
@@ -14,6 +13,7 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE


@notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
    try:
        current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -21,18 +21,11 @@ def deliver_sms(self, notification_id):
        if not notification:
            raise NoResultFound()
        send_to_providers.send_sms_to_provider(notification)
-    except Exception as e:
-        if isinstance(e, SmsClientResponseException):
-            current_app.logger.warning(
-                "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True
-            )
-        else:
+    except Exception:
+        try:
            current_app.logger.exception(
                "SMS notification delivery for id: {} failed".format(notification_id)
            )
-
-        try:
            if self.request.retries == 0:
                self.retry(queue=QueueNames.RETRY, countdown=0)
            else:
@@ -45,6 +38,7 @@ def deliver_sms(self, notification_id):


@notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
    try:
        current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
@@ -52,22 +46,14 @@ def deliver_email(self, notification_id):
        if not notification:
            raise NoResultFound()
        send_to_providers.send_email_to_provider(notification)
-    except EmailClientNonRetryableException as e:
-        current_app.logger.exception(
-            f"Email notification {notification_id} failed: {e}"
-        )
+    except InvalidEmailError as e:
+        current_app.logger.exception(e)
        update_notification_status_by_id(notification_id, 'technical-failure')
-    except Exception as e:
+    except Exception:
        try:
-            if isinstance(e, AwsSesClientThrottlingSendRateException):
-                current_app.logger.warning(
-                    f"RETRY: Email notification {notification_id} was rate limited by SES"
-                )
-            else:
-                current_app.logger.exception(
-                    f"RETRY: Email notification {notification_id} failed"
-                )
-
+            current_app.logger.exception(
+                "RETRY: Email notification {} failed".format(notification_id)
+            )
            self.retry(queue=QueueNames.RETRY)
        except self.MaxRetriesExceededError:
            message = "RETRY FAILED: Max retries reached. " \
--- a/app/celery/reporting_tasks.py
+++ b/app/celery/reporting_tasks.py
@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta

 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst

 from app import notify_celery
@@ -8,15 +9,19 @@ from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_billing_dao import (
    fetch_billing_data_for_day,
-    update_fact_billing,
+    update_fact_billing
+)
+from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    LETTER_TYPE,
 )
-from app.dao.fact_notification_status_dao import update_fact_notification_status
-from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE


@notify_celery.task(name="create-nightly-billing")
@cronitor("create-nightly-billing")
+@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
    # day_start is a datetime.date() object. e.g.
    # up to 4 days of data counting back from day_start is consolidated
@@ -25,105 +30,73 @@ def create_nightly_billing(day_start=None):
    else:
        # When calling the task its a string in the format of "YYYY-MM-DD"
        day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 10):
-        process_day = (day_start - timedelta(days=i)).isoformat()
+    for i in range(0, 4):
+        process_day = day_start - timedelta(days=i)

        create_nightly_billing_for_day.apply_async(
-            kwargs={'process_day': process_day},
+            kwargs={'process_day': process_day.isoformat()},
            queue=QueueNames.REPORTING
        )
-        current_app.logger.info(
-            f"create-nightly-billing task: create-nightly-billing-for-day task created for {process_day}"
-        )


@notify_celery.task(name="create-nightly-billing-for-day")
+@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
    process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
-    current_app.logger.info(
-        f'create-nightly-billing-for-day task for {process_day}: started'
-    )

    start = datetime.utcnow()
    transit_data = fetch_billing_data_for_day(process_day=process_day)
    end = datetime.utcnow()

-    current_app.logger.info(
-        f'create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds'
-    )
+    current_app.logger.info(f'create-nightly-billing-for-day {process_day} fetched in {(end - start).seconds} seconds')

    for data in transit_data:
        update_fact_billing(data, process_day)

    current_app.logger.info(
-        f"create-nightly-billing-for-day task for {process_day}: "
-        f"task complete. {len(transit_data)} rows updated"
+        f"create-nightly-billing-for-day task complete. {len(transit_data)} rows updated for day: {process_day}"
    )


@notify_celery.task(name="create-nightly-notification-status")
@cronitor("create-nightly-notification-status")
+@statsd(namespace="tasks")
 def create_nightly_notification_status():
-    """
-    Aggregate notification statuses into rows in ft_notification_status.
-    In order to minimise effort, this task assumes that:
-
-      - Email + SMS statuses don't change after 3 days. This is currently true
-        because all outstanding email / SMS are "timed out" after 3 days, and
-        we reject delivery receipts after this point.
-
-      - Letter statuses don't change after 9 days. There's no "timeout" for
-        letters but this is the longest we've had to cope with in the past - due
-        to major issues with our print provider.
-
-    Because the time range of the task exceeds the minimum possible retention
-    period (3 days), we need to choose which table to query for each service.
-
-    The aggregation happens for 1 extra day in case:
-
-      - This task or the "timeout" task fails to run.
-
-      - Data is (somehow) still in transit to the history table, which would
-        mean the aggregated results are temporarily incorrect.
-    """
-
    yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)

-    for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
-        days = 10 if notification_type == LETTER_TYPE else 4
-
-        for i in range(days):
-            process_day = yesterday - timedelta(days=i)
-
-            relevant_service_ids = get_service_ids_with_notifications_on_date(
-                notification_type, process_day
+    # email and sms
+    for i in range(4):
+        process_day = yesterday - timedelta(days=i)
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
            )
-
-            for service_id in relevant_service_ids:
-                create_nightly_notification_status_for_service_and_day.apply_async(
-                    kwargs={
-                        'process_day': process_day.isoformat(),
-                        'notification_type': notification_type,
-                        'service_id': service_id,
-                    },
-                    queue=QueueNames.REPORTING
-                )
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
+    for i in range(10):
+        process_day = yesterday - timedelta(days=i)
+        create_nightly_notification_status_for_day.apply_async(
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )


-@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
-def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
+@notify_celery.task(name="create-nightly-notification-status-for-day")
+@statsd(namespace="tasks")
+def create_nightly_notification_status_for_day(process_day, notification_type):
    process_day = datetime.strptime(process_day, "%Y-%m-%d").date()

    start = datetime.utcnow()
-    update_fact_notification_status(
-        process_day=process_day,
-        notification_type=notification_type,
-        service_id=service_id
-    )
-
+    transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
    end = datetime.utcnow()
    current_app.logger.info(
-        f'create-nightly-notification-status-for-service-and-day task update '
-        f'for {service_id}, {notification_type} for {process_day}: '
-        f'updated in {(end - start).seconds} seconds'
+        f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
+        f'data fetched in {(end - start).seconds} seconds'
+    )
+
+    update_fact_notification_status(transit_data, process_day, notification_type)
+
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
+        f'task complete - {len(transit_data)} rows updated'
    )
--- a/app/celery/research_mode_tasks.py
+++ b/app/celery/research_mode_tasks.py
@@ -1,16 +1,17 @@
-import json
 import random
 from datetime import datetime, timedelta
+import json

 from flask import current_app
+from requests import request, RequestException, HTTPError
+
 from notifications_utils.s3 import s3upload
-from requests import HTTPError, request

 from app import notify_celery
 from app.aws.s3 import file_exists
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
 from app.models import SMS_TYPE
+from app.config import QueueNames
+from app.celery.process_ses_receipts_tasks import process_ses_results

 temp_fail = "7700900003"
 perm_fail = "7700900002"
@@ -65,14 +66,16 @@ def make_request(notification_type, provider, data, headers):
            timeout=60
        )
        response.raise_for_status()
-    except HTTPError as e:
+    except RequestException as e:
+        api_error = HTTPError(e)
        current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
+            "API {} request on {} failed with {}".format(
+                "POST",
                api_call,
-                e.response.status_code
+                api_error.response
            )
        )
-        raise e
+        raise api_error
    finally:
        current_app.logger.info("Mocked provider callback request finished")
    return response.json()
@@ -164,7 +167,7 @@ def ses_notification_callback(reference):
            'processingTimeMillis': 2003,
            'recipients': ['success@simulator.amazonses.com'],
            'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
            'smtpResponse': '250 2.6.0 Message received',
            'timestamp': '2017-11-17T12:14:03.646Z'
        },
@@ -201,7 +204,7 @@ def ses_notification_callback(reference):
            'messageId': reference,
            'sendingAccountId': '12341234',
            'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
            'sourceIp': '0.0.0.1',
            'timestamp': '2017-11-17T12:14:01.643Z'
        },
@@ -211,14 +214,14 @@ def ses_notification_callback(reference):
    return {
        'Type': 'Notification',
        'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
        'Subject': None,
        'Message': json.dumps(ses_message_body),
        'Timestamp': '2017-11-17T12:14:03.710Z',
        'SignatureVersion': '1',
        'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
        'MessageAttributes': {}
    }

@@ -244,7 +247,7 @@ def _ses_bounce_callback(reference, bounce_type):
            }],
            'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
            'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
            'timestamp': '2017-11-17T12:14:05.131Z'
        },
        'mail': {
@@ -280,7 +283,7 @@ def _ses_bounce_callback(reference, bounce_type):
            'messageId': reference,
            'sendingAccountId': '12341234',
            'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
            'sourceIp': '0.0.0.1',
            'timestamp': '2017-11-17T12:14:03.000Z'
        },
@@ -289,13 +292,13 @@ def _ses_bounce_callback(reference, bounce_type):
    return {
        'Type': 'Notification',
        'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
        'Subject': None,
        'Message': json.dumps(ses_message_body),
        'Timestamp': '2017-11-17T12:14:05.149Z',
        'SignatureVersion': '1',
        'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
        'MessageAttributes': {}
    }
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -1,66 +1,57 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)

 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from sqlalchemy import between
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import and_
 from sqlalchemy.exc import SQLAlchemyError

-from app import db, notify_celery, zendesk_client
-from app.aws import s3
-from app.celery.broadcast_message_tasks import trigger_link_test
-from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app import notify_celery, zendesk_client
 from app.celery.tasks import (
-    get_recipient_csv_and_template_and_sender_id,
-    process_incomplete_jobs,
    process_job,
-    process_row,
+    get_recipient_csv_and_template_and_sender_id,
+    process_row
 )
+from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames, TaskNames
-from app.dao.invited_org_user_dao import (
-    delete_org_invitations_created_more_than_two_days_ago,
-)
-from app.dao.invited_user_dao import (
-    delete_invitations_created_more_than_two_days_ago,
-)
+from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
    dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
    find_jobs_with_missing_rows,
-    find_missing_row_for_job,
+    find_missing_row_for_job
 )
+from app.dao.jobs_dao import dao_update_job
 from app.dao.notifications_dao import (
-    dao_old_letters_with_created_status,
-    dao_precompiled_letters_still_pending_virus_check,
-    is_delivery_slow_for_providers,
-    letters_missing_from_sending_bucket,
+    dao_get_scheduled_notifications,
+    set_scheduled_notification_to_processed,
    notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket,
+    is_delivery_slow_for_providers,
 )
 from app.dao.provider_details_dao import (
-    dao_adjust_provider_priority_back_to_resting_points,
    dao_reduce_sms_provider_priority,
-)
-from app.dao.services_dao import (
-    dao_find_services_sending_to_tv_numbers,
-    dao_find_services_with_high_failure_rates,
+    dao_adjust_provider_priority_back_to_resting_points
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.letters.utils import generate_letter_pdf_filename
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
 from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
-    BroadcastMessage,
-    BroadcastStatusType,
    Job,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_ERROR,
+    SMS_TYPE,
+    EMAIL_TYPE,
 )
 from app.notifications.process_notifications import send_notification_to_queue
+from app.v2.errors import JobIncompleteError


@notify_celery.task(name="run-scheduled-jobs")
+@statsd(namespace="tasks")
 def run_scheduled_jobs():
    try:
        for job in dao_set_scheduled_jobs_to_pending():
@@ -71,7 +62,23 @@ def run_scheduled_jobs():
        raise


+@notify_celery.task(name='send-scheduled-notifications')
+@statsd(namespace="tasks")
+def send_scheduled_notifications():
+    try:
+        scheduled_notifications = dao_get_scheduled_notifications()
+        for notification in scheduled_notifications:
+            send_notification_to_queue(notification, notification.service.research_mode)
+            set_scheduled_notification_to_processed(notification.id)
+        current_app.logger.info(
+            "Sent {} scheduled notifications to the provider queue".format(len(scheduled_notifications)))
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to send scheduled notifications")
+        raise
+
+
@notify_celery.task(name="delete-verify-codes")
+@statsd(namespace="tasks")
 def delete_verify_codes():
    try:
        start = datetime.utcnow()
@@ -85,6 +92,7 @@ def delete_verify_codes():


@notify_celery.task(name="delete-invitations")
+@statsd(namespace="tasks")
 def delete_invitations():
    try:
        start = datetime.utcnow()
@@ -99,6 +107,7 @@ def delete_invitations():


@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
 def switch_current_sms_provider_on_slow_delivery():
    """
    Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
@@ -121,42 +130,32 @@ def switch_current_sms_provider_on_slow_delivery():


@notify_celery.task(name='tend-providers-back-to-middle')
+@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
    dao_adjust_provider_priority_back_to_resting_points()


@notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
 def check_job_status():
    """
    every x minutes do this check
    select
    from jobs
    where job_status == 'in progress'
-    and processing started between 30 and 35 minutes ago
-    OR where the job_status == 'pending'
-    and the job scheduled_for timestamp is between 30 and 35 minutes ago.
+    and template_type in ('sms', 'email')
+    and scheduled_at or created_at is older that 30 minutes.
    if any results then
-        update the job_status to 'error'
+        raise error
        process the rows in the csv that are missing (in another task) just do the check here.
    """
    thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
    thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)

-    incomplete_in_progress_jobs = Job.query.filter(
+    jobs_not_complete_after_30_minutes = Job.query.filter(
        Job.job_status == JOB_STATUS_IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-    incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
-        Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-
-    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
-        incomplete_pending_jobs
-    ).order_by(
-        Job.processing_started, Job.scheduled_for
-    ).all()
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
+    ).order_by(Job.processing_started).all()

    # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
    # if they haven't been re-processed in time.
@@ -167,14 +166,16 @@ def check_job_status():
        job_ids.append(str(job.id))

    if job_ids:
-        current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
-        process_incomplete_jobs.apply_async(
-            [job_ids],
+        notify_celery.send_task(
+            name=TaskNames.PROCESS_INCOMPLETE_JOBS,
+            args=(job_ids,),
            queue=QueueNames.JOBS
        )
+        raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))


@notify_celery.task(name='replay-created-notifications')
+@statsd(namespace="tasks")
 def replay_created_notifications():
    # if the notification has not be send after 1 hour, then try to resend.
    resend_created_notifications_older_than = (60 * 60)
@@ -204,91 +205,75 @@ def replay_created_notifications():

        current_app.logger.info(msg)
        for letter in letters:
-            get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)
+            create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)


-@notify_celery.task(name='check-if-letters-still-pending-virus-check')
-def check_if_letters_still_pending_virus_check():
-    letters = []
-
-    for letter in dao_precompiled_letters_still_pending_virus_check():
-        # find letter in the scan bucket
-        filename = generate_letter_pdf_filename(
-            letter.reference,
-            letter.created_at,
-            ignore_folder=True,
-            postage=letter.postage
-        )
-
-        if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename):
-            current_app.logger.warning(
-                f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.'
-            )
-            notify_celery.send_task(
-                name=TaskNames.SCAN_FILE,
-                kwargs={'filename': filename},
-                queue=QueueNames.ANTIVIRUS,
-            )
-        else:
-            letters.append(letter)
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()

    if len(letters) > 0:
-        letter_ids = [(str(letter.id), letter.reference) for letter in letters]
+        letter_ids = [str(letter.id) for letter in letters]

-        msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes.
-            We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off
-            again or move them to technical failure.
+        msg = "{} precompiled letters have been pending-virus-check for over 90 minutes. " \
+              "Notifications: {}".format(len(letters), letter_ids)

-            Notifications: {sorted(letter_ids)}"""
+        current_app.logger.exception(msg)

        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
                message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)


-@notify_celery.task(name='check-if-letters-still-in-created')
-def check_if_letters_still_in_created():
+@notify_celery.task(name='check-templated-letter-state')
+@statsd(namespace="tasks")
+def check_templated_letter_state():
+    # renaming task for clarity. Next deploy will remove this task.
+    check_letters_created_yesterday_are_sending()
+
+
+@notify_celery.task(name='check-letters-created-yesterday-are-sending')
+@statsd(namespace="tasks")
+def check_letters_created_yesterday_are_sending():
    letters = dao_old_letters_with_created_status()

    if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
        msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Follow runbook to resolve: " \
-              "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
-              "#deal-with-Letters-still-in-created.".format(len(letters))
+              "Notifications: {}".format(len(letters), letter_ids)
+
+        current_app.logger.exception(msg)

        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
                message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)


@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
 def check_for_missing_rows_in_completed_jobs():
-    jobs = find_jobs_with_missing_rows()
-    for job in jobs:
-        recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
+    jobs_and_job_size = find_jobs_with_missing_rows()
+    for x in jobs_and_job_size:
+        job = x[1]
        missing_rows = find_missing_row_for_job(job.id, job.notification_count)
        for row_to_process in missing_rows:
-            row = recipient_csv[row_to_process.missing_row]
-            current_app.logger.info(
-                "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
-            process_row(row, template, job, job.service, sender_id=sender_id)
+            recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
+            for row in recipient_csv.get_rows():
+                if row.index == row_to_process.missing_row:
+                    current_app.logger.info(
+                        "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
+                    process_row(row, template, job, job.service, sender_id=sender_id)


@notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
+@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
    start_date = (datetime.utcnow() - timedelta(days=1))
    end_date = datetime.utcnow()
@@ -326,44 +311,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
            message += ("\nYou can find instructions for this ticket in our manual:\n"
                        "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
+            zendesk_client.create_ticket(
+                subject="[{}] High failure rates for sms spotted for services".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']
+                ),
                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-@notify_celery.task(name='trigger-link-tests')
-def trigger_link_tests():
-    if current_app.config['CBC_PROXY_ENABLED']:
-        for cbc_name in current_app.config['ENABLED_CBCS']:
-            trigger_link_test.apply_async(kwargs={'provider': cbc_name}, queue=QueueNames.BROADCASTS)
-
-
-@notify_celery.task(name='auto-expire-broadcast-messages')
-def auto_expire_broadcast_messages():
-    expired_broadcasts = BroadcastMessage.query.filter(
-        BroadcastMessage.finishes_at <= datetime.now(),
-        BroadcastMessage.status == BroadcastStatusType.BROADCASTING,
-    ).all()
-
-    for broadcast in expired_broadcasts:
-        broadcast.status = BroadcastStatusType.COMPLETED
-
-    db.session.commit()
-
-    if expired_broadcasts:
-        notify_celery.send_task(
-            name=TaskNames.PUBLISH_GOVUK_ALERTS,
-            queue=QueueNames.GOVUK_ALERTS
-        )
-
-
-@notify_celery.task(name='remove-yesterdays-planned-tests-on-govuk-alerts')
-def remove_yesterdays_planned_tests_on_govuk_alerts():
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -1,14 +1,22 @@
 import json

 from flask import current_app
-from requests import HTTPError, RequestException, request
+from notifications_utils.statsd_decorators import statsd
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)

-from app import encryption, notify_celery
+from app import (
+    notify_celery,
+    encryption
+)
 from app.config import QueueNames
-from app.utils import DATETIME_FORMAT


@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_delivery_status_to_service(
    self, notification_id, encrypted_status_update
 ):
@@ -22,11 +30,8 @@ def send_delivery_status_to_service(
        "created_at": status_update['notification_created_at'],
        "completed_at": status_update['notification_updated_at'],
        "sent_at": status_update['notification_sent_at'],
-        "notification_type": status_update['notification_type'],
-        "template_id": status_update['template_id'],
-        "template_version": status_update['template_version']
+        "notification_type": status_update['notification_type']
    }
-
    _send_data_to_service_callback_api(
        self,
        data,
@@ -37,6 +42,7 @@ def send_delivery_status_to_service(


@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
    complaint = encryption.decrypt(complaint_data)

@@ -68,7 +74,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                'Content-Type': 'application/json',
                'Authorization': 'Bearer {}'.format(token)
            },
-            timeout=5
+            timeout=60
        )
        current_app.logger.info('{} sending {} to {}, response {}'.format(
            function_name,
@@ -79,16 +85,16 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
        response.raise_for_status()
    except RequestException as e:
        current_app.logger.warning(
-            "{} request failed for notification_id: {} and url: {}. exception: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
                function_name,
                notification_id,
                service_callback_url,
                e
            )
        )
-        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
            try:
-                self.retry(queue=QueueNames.CALLBACKS_RETRY)
+                self.retry(queue=QueueNames.RETRY)
            except self.MaxRetriesExceededError:
                current_app.logger.warning(
                    "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
@@ -97,18 +103,10 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                        notification_id
                    )
                )
-        else:
-            current_app.logger.warning(
-                "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name,
-                    notification_id,
-                    service_callback_url,
-                    e
-                )
-            )


 def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import DATETIME_FORMAT, encryption
    data = {
        "notification_id": str(notification.id),
        "notification_client_reference": notification.client_reference,
@@ -121,13 +119,12 @@ def create_delivery_status_callback_data(notification, service_callback_api):
        "notification_type": notification.notification_type,
        "service_callback_api_url": service_callback_api.url,
        "service_callback_api_bearer_token": service_callback_api.bearer_token,
-        "template_id": str(notification.template_id),
-        "template_version": notification.template_version,
    }
    return encryption.encrypt(data)


 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
    data = {
        "complaint_id": str(complaint.id),
        "notification_id": str(notification.id),
--- a/app/celery/tasks.py
+++ b/app/celery/tasks.py
@@ -1,38 +1,53 @@
 import json
-from collections import defaultdict, namedtuple
 from datetime import datetime
+from collections import namedtuple, defaultdict

 from flask import current_app
-from notifications_utils.insensitive_dict import InsensitiveDict
-from notifications_utils.postal_address import PostalAddress
-from notifications_utils.recipients import RecipientCSV
+from notifications_utils.recipients import (
+    RecipientCSV
+)
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.template import (
+    SMSMessageTemplate,
+    WithSubjectTemplate,
+)
 from notifications_utils.timezones import convert_utc_to_bst
-from requests import HTTPError, RequestException, request
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
+from sqlalchemy.exc import SQLAlchemyError

-from app import create_random_identifier, create_uuid, encryption, notify_celery
+from app import (
+    create_uuid,
+    create_random_identifier,
+    DATETIME_FORMAT,
+    encryption,
+    notify_celery,
+)
 from app.aws import s3
-from app.celery import letters_pdf_tasks, provider_tasks, research_mode_tasks
+from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
 from app.config import QueueNames
-from app.dao.daily_sorted_letter_dao import (
-    dao_create_or_update_daily_sorted_letter,
-)
+from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
-from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id,
+)
 from app.dao.notifications_dao import (
-    dao_get_last_notification_added_for_job_id,
-    dao_get_notification_or_history_by_reference,
-    dao_update_notifications_by_reference,
    get_notification_by_id,
+    dao_update_notifications_by_reference,
+    dao_get_last_notification_added_for_job_id,
    update_notification_status_by_reference,
+    dao_get_notification_history_by_reference,
 )
-from app.dao.provider_details_dao import (
-    get_provider_details_by_notification_type,
-)
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
 from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
@@ -46,22 +61,19 @@ from app.models import (
    LETTER_TYPE,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
-    NOTIFICATION_RETURNED_LETTER,
    NOTIFICATION_SENDING,
-    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
    SMS_TYPE,
    DailySortedLetter,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
-from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
-from app.v2.errors import TooManyRequestsError


@notify_celery.task(name="process-job")
+@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
    start = datetime.utcnow()
    job = dao_get_job_by_id(job_id)
@@ -72,10 +84,6 @@ def process_job(job_id, sender_id=None):

    service = job.service

-    job.job_status = JOB_STATUS_IN_PROGRESS
-    job.processing_started = start
-    dao_update_job(job)
-
    if not service.active:
        job.job_status = JOB_STATUS_CANCELLED
        dao_update_job(job)
@@ -86,6 +94,10 @@ def process_job(job_id, sender_id=None):
    if __sending_limits_for_job_exceeded(service, job, job_id):
        return

+    job.job_status = JOB_STATUS_IN_PROGRESS
+    job.processing_started = start
+    dao_update_job(job)
+
    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)

    current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
@@ -115,10 +127,13 @@ def job_complete(job, resumed=False, start=None):

 def get_recipient_csv_and_template_and_sender_id(job):
    db_template = dao_get_template_by_id(job.template_id, job.template_version)
-    template = db_template._as_utils_template()

+    TemplateClass = get_template_class(db_template.template_type)
+    template = TemplateClass(db_template.__dict__)
    contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id))
-    recipient_csv = RecipientCSV(contents, template=template)
+    recipient_csv = RecipientCSV(file_data=contents,
+                                 template_type=template.template_type,
+                                 placeholders=template.placeholders)

    return recipient_csv, template, meta_data.get("sender_id")

@@ -160,13 +175,9 @@ def process_row(row, template, job, service, sender_id=None):


 def __sending_limits_for_job_exceeded(service, job, job_id):
-    try:
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-        if total_sent + job.notification_count > service.message_limit:
-            raise TooManyRequestsError(service.message_limit)
-        else:
-            return False
-    except TooManyRequestsError:
+    total_sent = fetch_todays_total_message_count(service.id)
+
+    if total_sent + job.notification_count > service.message_limit:
        job.job_status = 'sending limits exceeded'
        job.processing_finished = datetime.utcnow()
        dao_update_job(job)
@@ -175,26 +186,24 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
                job_id, job.notification_count, service.message_limit)
        )
        return True
+    return False


@notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_sms(self,
             service_id,
             notification_id,
             encrypted_notification,
             sender_id=None):
    notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    if sender_id:
        reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
    else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()

    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
        current_app.logger.debug(
@@ -236,6 +245,7 @@ def save_sms(self,


@notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_email(self,
               service_id,
               notification_id,
@@ -243,17 +253,13 @@ def save_email(self,
               sender_id=None):
    notification = encryption.decrypt(encrypted_notification)

-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    if sender_id:
        reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
    else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()

    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
        current_app.logger.info("Email {} failed as restricted service".format(notification_id))
@@ -286,62 +292,8 @@ def save_email(self,
        handle_exception(self, notification, notification_id, e)


-@notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
-def save_api_email(self, encrypted_notification):
-
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
-def save_api_sms(self, encrypted_notification):
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-def save_api_email_or_sms(self, encrypted_notification):
-    notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification['service_id'])
-    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
-    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
-        else provider_tasks.deliver_sms
-    try:
-
-        persist_notification(
-            notification_id=notification["id"],
-            template_id=notification['template_id'],
-            template_version=notification['template_version'],
-            recipient=notification['to'],
-            service=service,
-            personalisation=notification.get('personalisation'),
-            notification_type=notification['notification_type'],
-            client_reference=notification['client_reference'],
-            api_key_id=notification.get('api_key_id'),
-            key_type=KEY_TYPE_NORMAL,
-            created_at=notification['created_at'],
-            reply_to_text=notification['reply_to_text'],
-            status=notification['status'],
-            document_download_count=notification['document_download_count']
-        )
-
-        q = q if not service.research_mode else QueueNames.RESEARCH_MODE
-        provider_task.apply_async(
-            [notification['id']],
-            queue=q
-        )
-        current_app.logger.debug(
-            f"{notification['notification_type']} {notification['id']} has been persisted and sent to delivery queue."
-        )
-    except IntegrityError:
-        current_app.logger.info(f"{notification['notification_type']} {notification['id']} already exists.")
-
-    except SQLAlchemyError:
-
-        try:
-            self.retry(queue=QueueNames.RETRY)
-        except self.MaxRetriesExceededError:
-            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
-
-
@notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_letter(
        self,
        service_id,
@@ -350,16 +302,11 @@ def save_letter(
 ):
    notification = encryption.decrypt(encrypted_notification)

-    postal_address = PostalAddress.from_personalisation(
-        InsensitiveDict(notification['personalisation'])
-    )
+    # we store the recipient as just the first item of the person's address
+    recipient = notification['personalisation']['addressline1']

-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    try:
        # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
@@ -368,8 +315,8 @@ def save_letter(
        saved_notification = persist_notification(
            template_id=notification['template'],
            template_version=notification['template_version'],
-            postage=postal_address.postage if postal_address.international else template.postage,
-            recipient=postal_address.normalised,
+            template_postage=template.postage,
+            recipient=recipient,
            service=service,
            personalisation=notification['personalisation'],
            notification_type=LETTER_TYPE,
@@ -380,13 +327,12 @@ def save_letter(
            job_row_number=notification['row_number'],
            notification_id=notification_id,
            reference=create_random_identifier(),
-            client_reference=get_reference_from_personalisation(notification['personalisation']),
-            reply_to_text=template.reply_to_text,
+            reply_to_text=template.get_reply_to_text(),
            status=status
        )

        if not service.research_mode:
-            letters_pdf_tasks.get_pdf_for_templated_letter.apply_async(
+            letters_pdf_tasks.create_letters_pdf.apply_async(
                [str(saved_notification.id)],
                queue=QueueNames.CREATE_LETTERS_PDF
            )
@@ -404,6 +350,7 @@ def save_letter(


@notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):
    # This task will be called by the FTP app to update notifications as sent to DVLA
    provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
@@ -422,6 +369,7 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):


@notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
    # This task will be called by the FTP app to update notifications as sent to DVLA

@@ -456,7 +404,17 @@ def handle_exception(task, notification, notification_id, exc):
            current_app.logger.error('Max retry failed' + retry_msg)


+def get_template_class(template_type):
+    if template_type == SMS_TYPE:
+        return SMSMessageTemplate
+    elif template_type in (EMAIL_TYPE, LETTER_TYPE):
+        # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
+        # use the same base template
+        return WithSubjectTemplate
+
+
@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
 def update_letter_notifications_statuses(self, filename):
    notification_updates = parse_dvla_file(filename)

@@ -473,6 +431,7 @@ def update_letter_notifications_statuses(self, filename):


@notify_celery.task(bind=True, name="record-daily-sorted-counts")
+@statsd(namespace="tasks")
 def record_daily_sorted_counts(self, filename):
    sorted_letter_counts = defaultdict(int)
    notification_updates = parse_dvla_file(filename)
@@ -547,7 +506,7 @@ def update_letter_notification(filename, temporary_failures, update):


 def check_billable_units(notification_update):
-    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_history_by_reference(notification_update.reference)

    if int(notification_update.page_count) != notification.billable_units:
        msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(
@@ -559,6 +518,7 @@ def check_billable_units(notification_update):


@notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
    inbound_api = get_service_inbound_api_for_service(service_id=service_id)
    if not inbound_api:
@@ -587,32 +547,36 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
            },
            timeout=60
        )
-        current_app.logger.debug(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
-            f"response {response.status_code}"
-        )
+        current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
+            inbound_sms_id,
+            inbound_api.url,
+            response.status_code
+        ))
        response.raise_for_status()
    except RequestException as e:
        current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
-            f"and url: {inbound_api.url}. exception: {e}"
+            "send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
+                service_id,
+                inbound_sms_id,
+                inbound_api.url,
+                e
+            )
        )
        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
            try:
                self.retry(queue=QueueNames.RETRY)
            except self.MaxRetriesExceededError:
                current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of" +
-                    f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
+                    """Retry: send_inbound_sms_to_service has retried the max number of
+                     times for service: {} and  inbound_sms {}""".format(
+                        service_id,
+                        inbound_sms_id
+                    )
                )
-        else:
-            current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
-                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
-            )


@notify_celery.task(name='process-incomplete-jobs')
+@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
    jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]

@@ -649,6 +613,7 @@ def process_incomplete_job(job_id):


@notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
 def process_returned_letters_list(notification_references):
    updated, updated_history = dao_update_notifications_by_reference(
        notification_references,
--- a/app/clients/init.py
+++ b/app/clients/init.py
@@ -1,6 +1,3 @@
-from celery import current_app
-
-
 class ClientException(Exception):
    '''
    Base Exceptions for sending notifications that fail
@@ -20,7 +17,7 @@ STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'


-class NotificationProviderClients(object):
+class Clients(object):
    sms_clients = {}
    email_clients = {}

@@ -39,7 +36,7 @@ class NotificationProviderClients(object):

    def get_client_by_name_and_type(self, name, notification_type):
        assert notification_type in ['email', 'sms']
-        
+
        if notification_type == 'email':
            return self.get_email_client(name)

--- a/app/clients/cbc_proxy.py
+++ b/app/clients/cbc_proxy.py
@@ -1,299 +0,0 @@
-import json
-import uuid
-from abc import ABC, abstractmethod
-
-import boto3
-import botocore
-from flask import current_app
-from notifications_utils.template import non_gsm_characters
-from sqlalchemy.schema import Sequence
-
-from app.config import BroadcastProvider
-from app.utils import DATETIME_FORMAT, format_sequential_number
-
-# The variable names in this file have specific meaning in a CAP message
-#
-# identifier is a unique field for each CAP message
-#
-# headline is a field which we are not sure if we will use
-#
-# description is the body of the message
-
-# areas is a list of dicts, with the following items
-# * description is a string which populates the areaDesc field
-# * polygon is a list of lat/long pairs
-#
-# previous_provider_messages is a list of previous events (models.py::BroadcastProviderMessage)
-# ie a Cancel message would have a unique event but have the event of
-#    the preceeding Alert message in the previous_provider_messages field
-
-
-class CBCProxyRetryableException(Exception):
-    pass
-
-
-class CBCProxyClient:
-    _lambda_client = None
-
-    def init_app(self, app):
-        if app.config.get('CBC_PROXY_ENABLED'):
-            self._lambda_client = boto3.client(
-                'lambda',
-                region_name='us-west-2',
-                aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
-                aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
-            )
-
-    def get_proxy(self, provider):
-        proxy_classes = {
-            BroadcastProvider.EE: CBCProxyEE,
-            BroadcastProvider.THREE: CBCProxyThree,
-            BroadcastProvider.O2: CBCProxyO2,
-            BroadcastProvider.VODAFONE: CBCProxyVodafone,
-        }
-        return proxy_classes[provider](self._lambda_client)
-
-
-class CBCProxyClientBase(ABC):
-    @property
-    @abstractmethod
-    def lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def failover_lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_ENGLISH(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_WELSH(self):
-        pass
-
-    def __init__(self, lambda_client):
-        self._lambda_client = lambda_client
-
-    def send_link_test(self):
-        self._send_link_test(self.lambda_name)
-        self._send_link_test(self.failover_lambda_name)
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ): pass
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        pass
-
-    # We have not implementated updating a broadcast
-    def update_and_send_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, channel, message_number=None
-    ):
-        pass
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, message_number=None
-    ):
-        pass
-
-    def _invoke_lambda_with_failover(self, payload):
-        result = self._invoke_lambda(self.lambda_name, payload)
-
-        if not result:
-            failover_result = self._invoke_lambda(self.failover_lambda_name, payload)
-            if not failover_result:
-                raise CBCProxyRetryableException(
-                    f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}'
-                )
-
-        return result
-
-    def _invoke_lambda(self, lambda_name, payload):
-        payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        try:
-            current_app.logger.info(
-                f"Calling lambda {lambda_name} with payload {str(payload)[:1000]}"
-            )
-
-            result = self._lambda_client.invoke(
-                FunctionName=lambda_name,
-                InvocationType='RequestResponse',
-                Payload=payload_bytes,
-            )
-        except botocore.exceptions.ClientError:
-            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
-            success = False
-            return success
-
-        if result['StatusCode'] > 299:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with status code { result['StatusCode']}, {result.get('Payload')}"
-            )
-            success = False
-
-        elif 'FunctionError' in result:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with function error { result['Payload'].read() }"
-            )
-            success = False
-
-        else:
-            success = True
-
-        return success
-
-    def infer_language_from(self, content):
-        if non_gsm_characters(content):
-            return self.LANGUAGE_WELSH
-        return self.LANGUAGE_ENGLISH
-
-
-class CBCProxyOne2ManyClient(CBCProxyClientBase):
-    LANGUAGE_ENGLISH = 'en-GB'
-    LANGUAGE_WELSH = 'cy-GB'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_format': 'cap'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_format': 'cap',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages,
-        sent, message_number=None
-    ):
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_format': 'cap',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-
-class CBCProxyEE(CBCProxyOne2ManyClient):
-    lambda_name = 'ee-1-proxy'
-    failover_lambda_name = 'ee-2-proxy'
-
-
-class CBCProxyThree(CBCProxyOne2ManyClient):
-    lambda_name = 'three-1-proxy'
-    failover_lambda_name = 'three-2-proxy'
-
-
-class CBCProxyO2(CBCProxyOne2ManyClient):
-    lambda_name = 'o2-1-proxy'
-    failover_lambda_name = 'o2-2-proxy'
-
-
-class CBCProxyVodafone(CBCProxyClientBase):
-    lambda_name = 'vodafone-1-proxy'
-    failover_lambda_name = 'vodafone-2-proxy'
-
-    LANGUAGE_ENGLISH = 'English'
-    LANGUAGE_WELSH = 'Welsh'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        from app import db
-        sequence = Sequence('broadcast_provider_message_number_seq')
-        sequential_number = db.session.connection().execute(sequence)
-        formatted_seq_number = format_sequential_number(sequential_number)
-
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_number': formatted_seq_number,
-            'message_format': 'ibag'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, message_number, headline, description, areas, sent, expires, channel
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self, identifier, previous_provider_messages, sent, message_number
-    ):
-
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "message_number": format_sequential_number(message.message_number),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
--- a/app/clients/document_download.py
+++ b/app/clients/document_download.py
@@ -1,4 +1,7 @@
+import base64
+
 import requests
+
 from flask import current_app


@@ -23,16 +26,15 @@ class DocumentDownloadClient:
    def get_upload_url(self, service_id):
        return "{}/services/{}/documents".format(self.api_host, service_id)

-    def upload_document(self, service_id, file_contents, is_csv=None):
+    def upload_document(self, service_id, file_contents):
        try:
            response = requests.post(
                self.get_upload_url(service_id),
                headers={
                    'Authorization': "Bearer {}".format(self.auth_token),
                },
-                json={
-                    'document': file_contents,
-                    'is_csv': is_csv or False,
+                files={
+                    'document': base64.b64decode(file_contents)
                }
            )

@@ -41,9 +43,7 @@ class DocumentDownloadClient:
            # if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
            # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
            # unrecognised) should be raised as a 500 internal server error here.
-            if e.response is None:
-                raise Exception(f'Unhandled document download error: {repr(e)}')
-            elif e.response.status_code == 400:
+            if e.response.status_code == 400:
                error = DocumentDownloadError.from_exception(e)
                current_app.logger.info(
                    'Document download request failed with error: {}'.format(error.message)
--- a/app/clients/email/init.py
+++ b/app/clients/email/init.py
@@ -1,4 +1,4 @@
-from app.clients import Client, ClientException
+from app.clients import ClientException, Client


 class EmailClientException(ClientException):
@@ -8,18 +8,6 @@ class EmailClientException(ClientException):
    pass


-class EmailClientNonRetryableException(ClientException):
-    '''
-    Represents an error returned from the email client API with a 4xx response code
-    that should not be retried and should instead be marked as technical failure.
-    An example of this would be an email address that makes it through our
-    validation rules but is rejected by SES. There is no point in retrying this type as
-    it will always fail however many calls to SES. Whereas a throttling error would not
-    use this exception as it may succeed if we retry
-    '''
-    pass
-
-
 class EmailClient(Client):
    '''
    Base Email client for sending emails.
@@ -28,6 +16,5 @@ class EmailClient(Client):
    def send_email(self, *args, **kwargs):
        raise NotImplementedError('TODO Need to implement.')

-    @property
-    def name(self):
+    def get_name(self):
        raise NotImplementedError('TODO Need to implement.')
--- a/app/clients/email/aws_ses.py
+++ b/app/clients/email/aws_ses.py
@@ -1,15 +1,11 @@
-from time import monotonic
-
 import boto3
 import botocore
 from flask import current_app
+from time import monotonic
+from notifications_utils.recipients import InvalidEmailError

 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
-from app.clients.email import (
-    EmailClient,
-    EmailClientException,
-    EmailClientNonRetryableException,
-)
+from app.clients.email import (EmailClientException, EmailClient)

 ses_response_map = {
    'Permanent': {
@@ -47,10 +43,6 @@ class AwsSesClientException(EmailClientException):
    pass


-class AwsSesClientThrottlingSendRateException(AwsSesClientException):
-    pass
-
-
 class AwsSesClient(EmailClient):
    '''
    Amazon SES email client.
@@ -59,11 +51,11 @@ class AwsSesClient(EmailClient):
    def init_app(self, region, statsd_client, *args, **kwargs):
        self._client = boto3.client('ses', region_name=region)
        super(AwsSesClient, self).__init__(*args, **kwargs)
+        self.name = 'ses'
        self.statsd_client = statsd_client

-    @property
-    def name(self):
-        return 'ses'
+    def get_name(self):
+        return self.name

    def send_email(self,
                   source,
@@ -108,12 +100,10 @@ class AwsSesClient(EmailClient):

            # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
            if e.response['Error']['Code'] == 'InvalidParameterValue':
-                raise EmailClientNonRetryableException(e.response['Error']['Message'])
-            elif (
-                e.response['Error']['Code'] == 'Throttling'
-                and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
-            ):
-                raise AwsSesClientThrottlingSendRateException(str(e))
+                raise InvalidEmailError('email: "{}" message: "{}"'.format(
+                    to_addresses[0],
+                    e.response['Error']['Message']
+                ))
            else:
                self.statsd_client.incr("clients.ses.error")
                raise AwsSesClientException(str(e))
--- a/app/clients/email/aws_ses_stub.py
+++ b/app/clients/email/aws_ses_stub.py
@@ -1,49 +0,0 @@
-import json
-from time import monotonic
-
-from flask import current_app
-from requests import request
-
-from app.clients.email import EmailClient, EmailClientException
-
-
-class AwsSesStubClientException(EmailClientException):
-    pass
-
-
-class AwsSesStubClient(EmailClient):
-    def init_app(self, region, statsd_client, stub_url):
-        self.statsd_client = statsd_client
-        self.url = stub_url
-
-    @property
-    def name(self):
-        return 'ses'
-
-    def send_email(self,
-                   source,
-                   to_addresses,
-                   subject,
-                   body,
-                   html_body='',
-                   reply_to_address=None):
-        try:
-            start_time = monotonic()
-            response = request(
-                "POST",
-                self.url,
-                data={"id": "dummy-data"},
-                timeout=60
-            )
-            response.raise_for_status()
-            response_json = json.loads(response.text)
-
-        except Exception as e:
-            self.statsd_client.incr("clients.ses_stub.error")
-            raise AwsSesStubClientException(str(e))
-        else:
-            elapsed_time = monotonic() - start_time
-            current_app.logger.info("AWS SES stub request finished in {}".format(elapsed_time))
-            self.statsd_client.timing("clients.ses_stub.request-time", elapsed_time)
-            self.statsd_client.incr("clients.ses_stub.success")
-            return response_json['MessageId']
--- a/app/clients/performance_platform/init.py
+++ b/app/clients/performance_platform/init.py
--- a/app/clients/performance_platform/performance_platform_client.py
+++ b/app/clients/performance_platform/performance_platform_client.py
@@ -1,8 +1,9 @@
 import base64
 import json

-import requests
 from flask import current_app
+import requests
+
 from notifications_utils.timezones import convert_utc_to_bst


--- a/app/clients/sms/init.py
+++ b/app/clients/sms/init.py
@@ -1,10 +1,10 @@
-from app.clients import Client, ClientException
+from app.clients import (Client, ClientException)


 class SmsClientResponseException(ClientException):
-    """
+    '''
    Base Exception for SmsClientsResponses
-    """
+    '''

    def __init__(self, message):
        self.message = message
@@ -14,15 +14,12 @@ class SmsClientResponseException(ClientException):


 class SmsClient(Client):
-    """
+    '''
    Base Sms client for sending smss.
-    """
-
-    def init_app(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+    '''

    def send_sms(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

    def get_name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
--- a/app/clients/sms/aws_sns.py
+++ b/app/clients/sms/aws_sns.py
@@ -1,89 +0,0 @@
-import re
-from time import monotonic
-
-import boto3
-import botocore
-import phonenumbers
-
-from app.clients.sms import SmsClient
-
-
-class AwsSnsClient(SmsClient):
-    """
-    AwsSns sms client
-    """
-
-    def init_app(self, current_app, statsd_client, *args, **kwargs):
-        self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"])
-        self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"])
-        super(SmsClient, self).__init__(*args, **kwargs)
-        self.current_app = current_app
-        self.statsd_client = statsd_client
-        self.long_code_regex = re.compile(r"^\+1\d{10}$")
-        
-    @property
-    def name(self):
-        return 'sns'
-
-    def get_name(self):
-        return 'sns'
-
-    def send_sms(self, to, content, reference, sender=None, international=False):
-        matched = False
-
-        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
-            matched = True
-            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
-
-            client = self._client
-            # See documentation
-            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
-            attributes = {
-                "AWS.SNS.SMS.SMSType": {
-                    "DataType": "String",
-                    "StringValue": "Transactional",
-                }
-            }
-
-            # If sending with a long code number, we need to use another AWS region
-            # and specify the phone number we want to use as the origination number
-            send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender)
-            if send_with_dedicated_phone_number:
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": sender,
-                }
-
-            # If the number is US based, we must use a US Toll Free number to send the message
-            country = phonenumbers.region_code_for_number(match.number)
-            if country == "US":
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
-                }
-
-            try:
-                start_time = monotonic()
-                response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
-            except botocore.exceptions.ClientError as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            except Exception as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            finally:
-                elapsed_time = monotonic() - start_time
-                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
-                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
-                self.statsd_client.incr("clients.sns.success")
-            return response["MessageId"]
-
-        if not matched:
-            self.statsd_client.incr("clients.sns.error")
-            self.current_app.logger.error("No valid numbers found in {}".format(to))
-            raise ValueError("No valid numbers found for SMS delivery")
-
-    def _send_with_dedicated_phone_number(self, sender):
-        return sender and re.match(self.long_code_regex, sender)
--- a/app/clients/sms/firetext.py
+++ b/app/clients/sms/firetext.py
@@ -0,0 +1,107 @@
+import json
+import logging
+
+from time import monotonic
+from requests import request, RequestException
+
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+logger = logging.getLogger(__name__)
+
+# Firetext will send a delivery receipt with three different status codes.
+# The `firetext_response` maps these codes to the notification statistics status and notification status.
+# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
+# the notification status to temporary-failure rather than permanent failure.
+#  See the code in the notification_dao.update_notifications_status_by_id
+firetext_responses = {
+    '0': 'delivered',
+    '1': 'permanent-failure',
+    '2': 'pending'
+}
+
+
+def get_firetext_responses(status):
+    return firetext_responses[status]
+
+
+class FiretextClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class FiretextClient(SmsClient):
+    '''
+    FireText sms client.
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('FIRETEXT_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'firetext'
+        self.url = current_app.config.get('FIRETEXT_URL')
+        self.statsd_client = statsd_client
+
+    def get_name(self):
+        return self.name
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.firetext.success")
+        else:
+            self.statsd_client.incr("clients.firetext.error")
+            self.current_app.logger.error(log_message)
+
+    def send_sms(self, to, content, reference, sender=None):
+
+        data = {
+            "apiKey": self.api_key,
+            "from": self.from_number if sender is None else sender,
+            "to": to.replace('+', ''),
+            "message": content,
+            "reference": reference
+        }
+
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.url,
+                data=data,
+                timeout=60
+            )
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+                if response.json()['code'] != 0:
+                    raise ValueError()
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise FiretextClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise FiretextClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
+            self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
+        return response
--- a/app/clients/sms/mmg.py
+++ b/app/clients/sms/mmg.py
@@ -0,0 +1,102 @@
+import json
+from time import monotonic
+from requests import (request, RequestException)
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+mmg_response_map = {
+    '2': 'permanent-failure',
+    '3': 'delivered',
+    '4': 'temporary-failure',
+    '5': 'permanent-failure'
+}
+
+
+def get_mmg_responses(status):
+    return mmg_response_map[status]
+
+
+class MMGClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class MMGClient(SmsClient):
+    '''
+    MMG sms client
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('MMG_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'mmg'
+        self.statsd_client = statsd_client
+        self.mmg_url = current_app.config.get('MMG_URL')
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.mmg_url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.mmg.success")
+        else:
+            self.statsd_client.incr("clients.mmg.error")
+            self.current_app.logger.error(log_message)
+
+    def get_name(self):
+        return self.name
+
+    def send_sms(self, to, content, reference, multi=True, sender=None):
+        data = {
+            "reqType": "BULK",
+            "MSISDN": to,
+            "msg": content,
+            "sender": self.from_number if sender is None else sender,
+            "cid": reference,
+            "multi": multi
+        }
+
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.mmg_url,
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Basic {}'.format(self.api_key)
+                },
+                timeout=60
+            )
+
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise MMGClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise MMGClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
+            self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
+
+        return response
--- a/app/cloudfoundry_config.py
+++ b/app/cloudfoundry_config.py
@@ -1,12 +1,21 @@
-import json
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
 import os
+import json


 def extract_cloudfoundry_config():
    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+    set_config_env_vars(vcap_services)

+
+def set_config_env_vars(vcap_services):
    # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
-                                                                                                      'postgresql')
-    # Redis config
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri']
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']
+
+    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
--- a/app/commands.py
+++ b/app/commands.py
@@ -1,33 +1,29 @@
 import csv
 import functools
-import itertools
-import os
 import uuid
 from datetime import datetime, timedelta
+from decimal import Decimal

 import click
 import flask
+import itertools
 from click_datetime import Datetime as click_dt
 from flask import current_app, json
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
-from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from notifications_utils.statsd_decorators import statsd

-from app import db
+from app import db, DATETIME_FORMAT, encryption
 from app.aws import s3
-from app.celery.letters_pdf_tasks import (
-    get_pdf_for_templated_letter,
-    resanitise_pdf,
-)
-from app.celery.tasks import process_row, record_daily_sorted_counts
+from app.celery.tasks import record_daily_sorted_counts, get_template_class, process_row
+from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
+from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.celery.letters_pdf_tasks import create_letters_pdf
+from app.celery.reporting_tasks import create_nightly_notification_status_for_day
 from app.config import QueueNames
-from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
-    set_default_free_allowance_for_service,
-)
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
 from app.dao.fact_billing_dao import (
    delete_billing_data_for_service_for_day,
    fetch_billing_data_for_day,
@@ -35,40 +31,35 @@ from app.dao.fact_billing_dao import (
    update_fact_billing,
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.dao.organisation_dao import (
-    dao_add_service_to_organisation,
-    dao_get_organisation_by_email_address,
-    dao_get_organisation_by_id,
-)
-from app.dao.permissions_dao import permission_dao
+from app.dao.organisation_dao import dao_get_organisation_by_email_address, dao_add_service_to_organisation
+
+from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.services_dao import (
-    dao_fetch_all_services_by_user,
-    dao_fetch_all_services_created_by_user,
-    dao_fetch_service_by_id,
-    dao_update_service,
    delete_service_and_all_associated_db_objects,
+    dao_fetch_all_services_by_user,
+    dao_fetch_service_by_id,
+    dao_update_service
 )
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.users_dao import (
-    delete_model_user,
-    delete_user_verify_codes,
-    get_user_by_email,
-)
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes, get_user_by_email
 from app.models import (
-    KEY_TYPE_TEST,
+    PROVIDERS,
    NOTIFICATION_CREATED,
+    KEY_TYPE_TEST,
    SMS_TYPE,
-    AnnualBilling,
-    Domain,
-    EmailBranding,
-    LetterBranding,
+    EMAIL_TYPE,
+    LETTER_TYPE,
+    User,
    Notification,
    Organisation,
-    Permission,
+    Domain,
    Service,
-    User,
+    EmailBranding,
+    LetterBranding,
 )
-from app.utils import get_london_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before


@click.group(name='command', help='Additional commands')
@@ -81,28 +72,32 @@ class notify_command:
        self.name = name

    def __call__(self, func):
-        decorators = [
-            functools.wraps(func),  # carry through function name, docstrings, etc.
-            click.command(name=self.name),  # turn it into a click.Command
-        ]
-
-        # in the test environment the app context is already provided and having
-        # another will lead to the test db connection being closed prematurely
-        if os.getenv('NOTIFY_ENVIRONMENT', '') != 'test':
-            # with_appcontext ensures the config is loaded, db connected, etc.
-            decorators.insert(0, flask.cli.with_appcontext)
-
+        # we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
+        # we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
+        # Then we need to turn it into a click.Command - that's what command_group.add_command expects.
+        @click.command(name=self.name)
+        @functools.wraps(func)
+        @flask.cli.with_appcontext
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)

-        for decorator in decorators:
-            # this syntax is equivalent to e.g. "@flask.cli.with_appcontext"
-            wrapper = decorator(wrapper)
-
        command_group.add_command(wrapper)
+
        return wrapper


+@notify_command()
+@click.option('-p', '--provider_name', required=True, type=click.Choice(PROVIDERS))
+@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals', type=float)
+@click.option('-d', '--valid_from', required=True, type=click_dt(format='%Y-%m-%dT%H:%M:%S'))
+def create_provider_rates(provider_name, cost, valid_from):
+    """
+    Backfill rates for a given provider
+    """
+    cost = Decimal(cost)
+    dao_create_provider_rates(provider_name, valid_from, cost)
+
+
@notify_command()
@click.option('-u', '--user_email_prefix', required=True, help="""
    Functional test user email prefix. eg "notify-test-preview"
@@ -111,7 +106,7 @@ def purge_functional_test_data(user_email_prefix):
    """
    Remove non-seeded functional test data

-    users, services, etc. Give an email prefix. Probably "notify-tests-preview".
+    users, services, etc. Give an email prefix. Probably "notify-test-preview".
    """
    users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
    for usr in users:
@@ -124,20 +119,9 @@ def purge_functional_test_data(user_email_prefix):
        else:
            services = dao_fetch_all_services_by_user(usr.id)
            if services:
-                print(f"Deleting user {usr.id} which is part of services")
                for service in services:
                    delete_service_and_all_associated_db_objects(service)
            else:
-                services_created_by_this_user = dao_fetch_all_services_created_by_user(usr.id)
-                if services_created_by_this_user:
-                    # user is not part of any services but may still have been the one to create the service
-                    # sometimes things get in this state if the tests fail half way through
-                    # Remove the service they created (but are not a part of) so we can then remove the user
-                    print(f"Deleting services created by {usr.id}")
-                    for service in services_created_by_this_user:
-                        delete_service_and_all_associated_db_objects(service)
-
-                print(f"Deleting user {usr.id} which is not part of any services")
                delete_user_verify_codes(usr)
                delete_model_user(usr)

@@ -223,43 +207,255 @@ def fix_notification_statuses_not_in_sync():
        result = db.session.execute(subq_hist).fetchall()


+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+
+    WARNING: This does not overwrite existing data. You need to delete
+             the existing data or Performance Platform will double-count.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_processing_time(start_date, end_date):
+    """
+    Send historical processing time to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+        # because the tz conversion funcs talk about midnight, and the midnight before last,
+        # we want to pretend we're running this from the next morning, so add one.
+        process_date = start_date + timedelta(days=i + 1)
+
+        process_start_date = get_midnight_for_day_before(process_date)
+        process_end_date = get_london_midnight_in_utc(process_date)
+
+        print('Sending notification processing-time for {} - {}'.format(
+            process_start_date.isoformat(),
+            process_end_date.isoformat()
+        ))
+        send_processing_time_for_start_and_end(process_start_date, process_end_date)
+
+
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='list-routes')
+def list_routes():
+    """List URLs of all application routes."""
+    for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
+        print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
+
+
@notify_command(name='insert-inbound-numbers')
@click.option('-f', '--file_name', required=True,
              help="""Full path of the file to upload, file is a contains inbound numbers,
              one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
    print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    file = open(file_name)
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"

-        for line in file:
-            line = line.strip()
-            if line:
-                print(line)
-                db.session.execute(sql.format(uuid.uuid4(), line))
-                db.session.commit()
+    for line in file:
+        print(line)
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
+        db.session.commit()
+    file.close()


-@notify_command(name='replay-create-pdf-for-templated-letter')
+@notify_command(name='replay-create-pdf-letters')
@click.option('-n', '--notification_id', type=click.UUID, required=True,
-              help="Notification id of the letter that needs the get_pdf_for_templated_letter task replayed")
-def replay_create_pdf_for_templated_letter(notification_id):
-    print("Create task to get_pdf_for_templated_letter for notification: {}".format(notification_id))
-    get_pdf_for_templated_letter.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)
+              help="Notification id of the letter that needs the create_letters_pdf task replayed")
+def replay_create_pdf_letters(notification_id):
+    print("Create task to create_letters_pdf for notification: {}".format(notification_id))
+    create_letters_pdf.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)


-@notify_command(name='recreate-pdf-for-precompiled-or-uploaded-letter')
-@click.option('-n', '--notification_id', type=click.UUID, required=True,
-              help="Notification ID of the precompiled or uploaded letter")
-def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id):
-    print(f"Call resanitise_pdf task for notification: {notification_id}")
-    resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS)
+@notify_command(name='replay-service-callbacks')
+@click.option('-f', '--file_name', required=True,
+              help="""Full path of the file to upload, file is a contains client references of
+              notifications that need the status to be sent to the service.""")
+@click.option('-s', '--service_id', required=True,
+              help="""The service that the callbacks are for""")
+def replay_service_callbacks(file_name, service_id):
+    print("Start send service callbacks for service: ", service_id)
+    callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
+    if not callback_api:
+        print("Callback api was not found for service: {}".format(service_id))
+        return
+
+    errors = []
+    notifications = []
+    file = open(file_name)
+
+    for ref in file:
+        try:
+            notification = Notification.query.filter_by(client_reference=ref.strip()).one()
+            notifications.append(notification)
+        except NoResultFound:
+            errors.append("Reference: {} was not found in notifications.".format(ref))
+
+    for e in errors:
+        print(e)
+    if errors:
+        raise Exception("Some notifications for the given references were not found")
+
+    for n in notifications:
+        data = {
+            "notification_id": str(n.id),
+            "notification_client_reference": n.client_reference,
+            "notification_to": n.to,
+            "notification_status": n.status,
+            "notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
+            "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
+            "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
+            "notification_type": n.notification_type,
+            "service_callback_api_url": callback_api.url,
+            "service_callback_api_bearer_token": callback_api.bearer_token,
+        }
+        encrypted_status_update = encryption.encrypt(data)
+        send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
+                                                    queue=QueueNames.CALLBACKS)
+
+    print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
+        service_id, len(notifications)))


 def setup_commands(application):
    application.cli.add_command(command_group)


+@notify_command(name='migrate-data-to-ft-billing')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@statsd(namespace="tasks")
+def migrate_data_to_ft_billing(start_date, end_date):
+
+    current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
+
+    process_date = start_date
+    total_updated = 0
+
+    while process_date < end_date:
+        start_time = datetime.utcnow()
+        # migrate data into ft_billing, upserting the data if it the record already exists
+        sql = \
+            """
+            insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
+                international, billable_units, notifications_sent, rate, postage, created_at)
+                select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
+                    case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
+                from (
+                    select
+                        n.id,
+                        (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
+                        coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
+                        coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
+                        n.notification_type,
+                        coalesce(n.sent_by, (
+                        case
+                        when notification_type = 'sms' then
+                            coalesce(sent_by, 'unknown')
+                        when notification_type = 'letter' then
+                            coalesce(sent_by, 'dvla')
+                        else
+                            coalesce(sent_by, 'ses')
+                        end )) as provider,
+                        coalesce(n.rate_multiplier,1) as rate_multiplier,
+                        s.crown,
+                        coalesce((select rates.rate from rates
+                        where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
+                        order by rates.valid_from desc limit 1), 0) as sms_rate,
+                        coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
+                        and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
+                        and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
+                        as letter_rate,
+                        coalesce(n.international, false) as international,
+                        n.billable_units,
+                        1 as notifications_sent,
+                        coalesce(n.postage, 'none') as postage,
+                        now() as created_at
+                    from public.notification_history n
+                    left join services s on s.id = n.service_id
+                    where n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
+                        and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
+                        at time zone 'UTC'
+                        and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
+                    ) as individual_record
+                group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sms_rate, letter_rate, postage, created_at
+                order by bst_date
+            on conflict on constraint ft_billing_pkey do update set
+             billable_units = excluded.billable_units,
+             notifications_sent = excluded.notifications_sent,
+             rate = excluded.rate,
+             updated_at = now()
+            """
+
+        result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
+        db.session.commit()
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
+
+        process_date += timedelta(days=1)
+
+        total_updated += result.rowcount
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
@notify_command(name='rebuild-ft-billing-for-day')
@click.option('-s', '--service_id', required=False, type=click.UUID)
@click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -299,6 +495,29 @@ def rebuild_ft_billing_for_day(service_id, day):
            rebuild_ft_data(day, row.service_id)


+@notify_command(name='migrate-data-to-ft-notification-status')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
+@statsd(namespace="tasks")
+def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
+    notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
+
+    start_date = start_date.date()
+    end_date = end_date.date()
+    for day_diff in range((end_date - start_date).days + 1):
+        process_day = start_date + timedelta(days=day_diff)
+        for notification_type in notification_types:
+            print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
+                process_day,
+                notification_type
+            ))
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
+            )
+
+
@notify_command(name='bulk-invite-user-to-service')
@click.option('-f', '--file_name', required=True,
              help="Full path of the file containing a list of email address for people to invite to a service")
@@ -315,7 +534,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi
    #  platform_admin
    #  view_activity
    # "send_texts,send_emails,send_letters,view_activity"
-    from app.service_invite.rest import create_invited_user
+    from app.invite.rest import create_invited_user
    file = open(file_name)
    for email_address in file:
        data = {
@@ -522,39 +741,6 @@ def populate_organisations_from_file(file_name):
                        db.session.rollback()


-@notify_command(name='populate-organisation-agreement-details-from-file')
-@click.option('-f', '--file_name', required=True,
-              help="CSV file containing id, agreement_signed_version, "
-              "agreement_signed_on_behalf_of_name, agreement_signed_at")
-def populate_organisation_agreement_details_from_file(file_name):
-    """
-    The input file should be a comma separated CSV file with a header row and 4 columns
-    id: the organisation ID
-    agreement_signed_version
-    agreement_signed_on_behalf_of_name
-    agreement_signed_at: The date the agreement was signed in the format of 'dd/mm/yyyy'
-    """
-    with open(file_name) as f:
-        csv_reader = csv.reader(f)
-
-        # ignore the header row
-        next(csv_reader)
-
-        for row in csv_reader:
-            org = dao_get_organisation_by_id(row[0])
-
-            current_app.logger.info(f"Updating {org.name}")
-
-            assert org.agreement_signed
-
-            org.agreement_signed_version = float(row[1])
-            org.agreement_signed_on_behalf_of_name = row[2].strip()
-            org.agreement_signed_at = datetime.strptime(row[3], "%d/%m/%Y")
-
-            db.session.add(org)
-            db.session.commit()
-
-
@notify_command(name='get-letter-details-from-zips-sent-file')
@click.argument('file_paths', required=True, nargs=-1)
@statsd(namespace="tasks")
@@ -575,24 +761,9 @@ def get_letter_details_from_zips_sent_file(file_paths):
        rows_from_file.extend(json.loads(file_contents))

    notification_references = tuple(row[18:34] for row in rows_from_file)
-    get_letters_data_from_references(notification_references)

-
-@notify_command(name='get-notification-and-service-ids-for-letters-that-failed-to-print')
-@click.option('-f', '--file_name', required=True,
-              help="""Full path of the file to upload, file should contain letter filenames, one per line""")
-def get_notification_and_service_ids_for_letters_that_failed_to_print(file_name):
-    print("Getting service and notification ids for letter filenames list {}".format(file_name))
-    file = open(file_name)
-    references = tuple([row[7:23] for row in file])
-
-    get_letters_data_from_references(tuple(references))
-    file.close()
-
-
-def get_letters_data_from_references(notification_references):
    sql = """
-        SELECT id, service_id, template_id, reference, job_id, created_at
+        SELECT id, service_id, reference, job_id, created_at
        FROM notifications
        WHERE reference IN :notification_references
        ORDER BY service_id, job_id"""
@@ -600,7 +771,7 @@ def get_letters_data_from_references(notification_references):

    with open('zips_sent_details.csv', 'w') as csvfile:
        csv_writer = csv.writer(csvfile)
-        csv_writer.writerow(['notification_id', 'service_id', 'template_id', 'reference', 'job_id', 'created_at'])
+        csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at'])

        for row in result:
            csv_writer.writerow(row)
@@ -718,7 +889,8 @@ def process_row_from_job(job_id, job_row_number):
    job = dao_get_job_by_id(job_id)
    db_template = dao_get_template_by_id(job.template_id, job.template_version)

-    template = db_template._as_utils_template()
+    TemplateClass = get_template_class(db_template.template_type)
+    template = TemplateClass(db_template.__dict__)

    for row in RecipientCSV(
            s3.get_job_from_s3(str(job.service_id), str(job.id)),
@@ -729,115 +901,3 @@ def process_row_from_job(job_id, job_row_number):
            notification_id = process_row(row, template, job, job.service)
            current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                job_row_number, job_id, notification_id))
-
-
-@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing_with_the_previous_years_allowance(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
-@notify_command(name='populate-annual-billing-with-defaults')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2021""")
-@click.option('-m', '--missing-services-only', default=True, type=bool,
-              help="""If true then only populate services missing from annual billing for the year.
-                      If false populate the default values for all active services.""")
-def populate_annual_billing_with_defaults(year, missing_services_only):
-    """
-    Add or update annual billing with free allowance defaults for all active services.
-    The default free allowance limits are in: app/dao/annual_billing_dao.py:57.
-
-    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
-    This is useful to prevent overriding any services that have a free allowance that is not the default.
-
-    If missing_services_only is false then add or update annual billing for all active services.
-    This is useful to ensure all services start the new year with the correct annual billing.
-    """
-    if missing_services_only:
-        active_services = Service.query.filter(
-            Service.active
-        ).outerjoin(
-            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
-        ).filter(
-            AnnualBilling.id == None  # noqa
-        ).all()
-    else:
-        active_services = Service.query.filter(
-            Service.active
-        ).all()
-    previous_year = year - 1
-    services_with_zero_free_allowance = db.session.query(AnnualBilling.service_id).filter(
-        AnnualBilling.financial_year_start == previous_year,
-        AnnualBilling.free_sms_fragment_limit == 0
-    ).all()
-
-    for service in active_services:
-
-        # If a service has free_sms_fragment_limit for the previous year
-        # set the free allowance for this year to 0 as well.
-        # Else use the default free allowance for the service.
-        if service.id in [x.service_id for x in services_with_zero_free_allowance]:
-            print(f'update service {service.id} to 0')
-            dao_create_or_update_annual_billing_for_year(
-                service_id=service.id,
-                free_sms_fragment_limit=0,
-                financial_year_start=year
-            )
-        else:
-            print(f'update service {service.id} with default')
-            set_default_free_allowance_for_service(service, year)
-
-
-@click.option('-u', '--user-id', required=True)
-@notify_command(name='local-dev-broadcast-permissions')
-def local_dev_broadcast_permissions(user_id):
-    if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
-    user = User.query.filter_by(id=user_id).one()
-
-    user_broadcast_services = Service.query.filter(
-        Service.permissions.any(permission='broadcast'),
-        Service.users.any(id=user_id)
-    )
-
-    for service in user_broadcast_services:
-        permission_list = [
-            Permission(service_id=service.id, user_id=user_id, permission=permission)
-            for permission in [
-                'reject_broadcasts', 'cancel_broadcasts',  # required to create / approve
-                'create_broadcasts', 'approve_broadcasts',  # minimum for testing
-                'manage_templates',  # unlikely but might be useful
-                'view_activity',  # normally added on invite / service creation
-            ]
-        ]
-
-        permission_dao.set_user_service_permission(
-            user, service, permission_list, _commit=True, replace=True
-        )
--- a/app/complaint/complaint_rest.py
+++ b/app/complaint/complaint_rest.py
@@ -3,10 +3,7 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request

 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import (
-    fetch_count_of_complaints,
-    fetch_paginated_complaints,
-)
+from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
 from app.errors import register_errors
 from app.schema_validation import validate
 from app.utils import pagination_links
--- a/app/complaint/complaint_schema.py
+++ b/app/complaint/complaint_schema.py
@@ -1,6 +1,6 @@

 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "complaint count request schema",
    "type": "object",
    "title": "Complaint count request",
--- a/app/config.py
+++ b/app/config.py
@@ -1,6 +1,6 @@
-import json
-import os
 from datetime import timedelta
+import os
+import json

 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -27,15 +27,9 @@ class QueueNames(object):
    PROCESS_FTP = 'process-ftp-tasks'
    CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
    CALLBACKS = 'service-callbacks'
-    CALLBACKS_RETRY = 'service-callbacks-retry'
    LETTERS = 'letter-tasks'
-    SMS_CALLBACKS = 'sms-callbacks'
    ANTIVIRUS = 'antivirus-tasks'
    SANITISE_LETTERS = 'sanitise-letter-tasks'
-    SAVE_API_EMAIL = 'save-api-email-tasks'
-    SAVE_API_SMS = 'save-api-sms-tasks'
-    BROADCASTS = 'broadcast-tasks'
-    GOVUK_ALERTS = 'govuk-alerts'

    @staticmethod
    def all_queues():
@@ -52,83 +46,59 @@ class QueueNames(object):
            QueueNames.NOTIFY,
            QueueNames.CREATE_LETTERS_PDF,
            QueueNames.CALLBACKS,
-            QueueNames.CALLBACKS_RETRY,
            QueueNames.LETTERS,
-            QueueNames.SMS_CALLBACKS,
-            QueueNames.SAVE_API_EMAIL,
-            QueueNames.SAVE_API_SMS,
-            QueueNames.BROADCASTS,
        ]


-class BroadcastProvider:
-    EE = 'ee'
-    VODAFONE = 'vodafone'
-    THREE = 'three'
-    O2 = 'o2'
-
-    PROVIDERS = [EE, VODAFONE, THREE, O2]
-
-
 class TaskNames(object):
    PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
    ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
    SCAN_FILE = 'scan-file'
    SANITISE_LETTER = 'sanitise-and-upload-letter'
-    CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter'
-    PUBLISH_GOVUK_ALERTS = 'publish-govuk-alerts'
-    RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter'


 class Config(object):
    # URL of admin app
-    ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL')
+    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')

    # URL of api app (on AWS this is the internal api endpoint)
-    API_HOST_NAME = os.environ.get('API_HOST_NAME')
+    API_HOST_NAME = os.getenv('API_HOST_NAME')

    # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = 'notify-admin'
-    GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove?
-
-    INTERNAL_CLIENT_API_KEYS = json.loads(
-        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}')
-    ) # TODO: handled by varsfile?
+    API_INTERNAL_SECRETS = json.loads(os.environ.get('API_INTERNAL_SECRETS', '[]'))

    # encyption secret/salt
-    ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
-    SECRET_KEY = os.environ.get('SECRET_KEY')
-    DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT')
+    SECRET_KEY = os.getenv('SECRET_KEY')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')

    # DB conection string
-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
-    
-    # AWS SMS
-    AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION")
-    AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER")
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')

    # MMG API Key
-    MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder')
+    MMG_API_KEY = os.getenv('MMG_API_KEY')

    # Firetext API Key
-    FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
-    FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
+    FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")

    # Prefix to identify queues in SQS
-    NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX')
+    NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')

    # URL of redis instance
-    REDIS_URL = os.environ.get('REDIS_URL')
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
+    REDIS_URL = os.getenv('REDIS_URL')
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
    EXPIRE_CACHE_TEN_MINUTES = 600
    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60

+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = False
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
+
    # Zendesk
    ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')

    # Logging
    DEBUG = False
-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH')
+    NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')

    # Cronitor
    CRONITOR_ENABLED = False
@@ -142,7 +112,8 @@ class Config(object):
    ###########################

    NOTIFY_ENVIRONMENT = 'development'
-    AWS_REGION = 'us-west-2'
+    ADMIN_CLIENT_USER_NAME = 'notify-admin'
+    AWS_REGION = 'eu-west-1'
    INVITATION_EXPIRATION_DAYS = 2
    NOTIFY_APP_NAME = 'api'
    SQLALCHEMY_RECORD_QUERIES = False
@@ -155,10 +126,7 @@ class Config(object):
    API_PAGE_SIZE = 250
    TEST_MESSAGE_FILENAME = 'Test message'
    ONE_OFF_MESSAGE_FILENAME = 'Report'
-    MAX_VERIFY_CODE_COUNT = 5
-    MAX_FAILED_LOGIN_COUNT = 10
-    
-    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
+    MAX_VERIFY_CODE_COUNT = 10

    # be careful increasing this size without being sure that we won't see slowness in pysftp
    MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
@@ -168,14 +136,13 @@ class Config(object):

    # these should always add up to 100%
    SMS_PROVIDER_RESTING_POINTS = {
-        'mmg': 50,
-        'firetext': 50
+        'mmg': 60,
+        'firetext': 40
    }

    NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
    NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
    INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
-    BROADCAST_INVITATION_EMAIL_TEMPLATE_ID = '46152f7c-6901-41d5-8590-a5624d0d4359'
    SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
    EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
    NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
@@ -190,171 +157,158 @@ class Config(object):
    MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
    MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
    MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
-    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
-    # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
+    MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'

-    CELERY = {
-        'broker_url': 'sqs://',
-        'broker_transport_options': {
-            'region': AWS_REGION,
-            'visibility_timeout': 310,
-            'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
-        },
-        'timezone': 'Europe/London',
-        'imports': [
-            'app.celery.tasks',
-            'app.celery.scheduled_tasks',
-            'app.celery.reporting_tasks',
-            'app.celery.nightly_tasks',
-        ],
-        # this is overriden by the -Q command, but locally, we should read from all queues
-        'task_queues': [
-            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
-        ],
-        'beat_schedule': {
-            # app/celery/scheduled_tasks.py
-            'run-scheduled-jobs': {
-                'task': 'run-scheduled-jobs',
-                'schedule': crontab(minute='0,15,30,45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-verify-codes': {
-                'task': 'delete-verify-codes',
-                'schedule': timedelta(minutes=63),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-invitations': {
-                'task': 'delete-invitations',
-                'schedule': timedelta(minutes=66),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'switch-current-sms-provider-on-slow-delivery': {
-                'task': 'switch-current-sms-provider-on-slow-delivery',
-                'schedule': crontab(),  # Every minute
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-job-status': {
-                'task': 'check-job-status',
-                'schedule': crontab(),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'tend-providers-back-to-middle': {
-                'task': 'tend-providers-back-to-middle',
-                'schedule': crontab(minute='*/5'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-missing-rows-in-completed-jobs': {
-                'task': 'check-for-missing-rows-in-completed-jobs',
-                'schedule': crontab(minute='*/10'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'replay-created-notifications': {
-                'task': 'replay-created-notifications',
-                'schedule': crontab(minute='0, 15, 30, 45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # app/celery/nightly_tasks.py
-            'timeout-sending-notifications': {
-                'task': 'timeout-sending-notifications',
-                'schedule': crontab(hour=0, minute=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'create-nightly-billing': {
-                'task': 'create-nightly-billing',
-                'schedule': crontab(hour=0, minute=15),
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'create-nightly-notification-status': {
-                'task': 'create-nightly-notification-status',
-                'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-notifications-older-than-retention': {
-                'task': 'delete-notifications-older-than-retention',
-                'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-inbound-sms': {
-                'task': 'delete-inbound-sms',
-                'schedule': crontab(hour=1, minute=40),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'save-daily-notification-processing-time': {
-                'task': 'save-daily-notification-processing-time',
-                'schedule': crontab(hour=2, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove_sms_email_jobs': {
-                'task': 'remove_sms_email_jobs',
-                'schedule': crontab(hour=4, minute=0),
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'remove_letter_jobs': {
-                'task': 'remove_letter_jobs',
-                'schedule': crontab(hour=4, minute=20),
-                # since we mark jobs as archived
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'check-if-letters-still-in-created': {
-                'task': 'check-if-letters-still-in-created',
-                'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-if-letters-still-pending-virus-check': {
-                'task': 'check-if-letters-still-pending-virus-check',
-                'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
-                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
-                'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-letter-notifications-still-sending': {
-                'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=17, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # The collate-letter-pdf does assume it is called in an hour that BST does not make a
-            # difference to the truncate date which translates to the filename to process
-            'collate-letter-pdfs-to-be-sent': {
-                'task': 'collate-letter-pdfs-to-be-sent',
-                'schedule': crontab(hour=17, minute=50),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-no-letter-ack-file': {
-                'task': 'raise-alert-if-no-letter-ack-file',
-                'schedule': crontab(hour=23, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'trigger-link-tests': {
-                'task': 'trigger-link-tests',
-                'schedule': timedelta(minutes=15),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'auto-expire-broadcast-messages': {
-                'task': 'auto-expire-broadcast-messages',
-                'schedule': timedelta(minutes=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove-yesterdays-planned-tests-on-govuk-alerts': {
-                'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
-                'schedule': crontab(hour=00, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-        }
+    BROKER_URL = 'sqs://'
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 310,
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
    }
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = 'Europe/London'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TASK_SERIALIZER = 'json'
+    CELERY_IMPORTS = (
+        'app.celery.tasks',
+        'app.celery.scheduled_tasks',
+        'app.celery.reporting_tasks',
+        'app.celery.nightly_tasks',
+    )
+    CELERYBEAT_SCHEDULE = {
+        # app/celery/scheduled_tasks.py
+        'run-scheduled-jobs': {
+            'task': 'run-scheduled-jobs',
+            'schedule': crontab(minute=1),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-verify-codes': {
+            'task': 'delete-verify-codes',
+            'schedule': timedelta(minutes=63),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-invitations': {
+            'task': 'delete-invitations',
+            'schedule': timedelta(minutes=66),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-job-status': {
+            'task': 'check-job-status',
+            'schedule': crontab(),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'tend-providers-back-to-middle': {
+            'task': 'tend-providers-back-to-middle',
+            'schedule': crontab(minute='*/5'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-missing-rows-in-completed-jobs': {
+            'task': 'check-for-missing-rows-in-completed-jobs',
+            'schedule': crontab(minute='*/10'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'replay-created-notifications': {
+            'task': 'replay-created-notifications',
+            'schedule': crontab(minute='0, 15, 30, 45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # app/celery/nightly_tasks.py
+        'timeout-sending-notifications': {
+            'task': 'timeout-sending-notifications',
+            'schedule': crontab(hour=0, minute=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=0, minute=15),
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'create-nightly-notification-status': {
+            'task': 'create-nightly-notification-status',
+            'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'delete-sms-notifications': {
+            'task': 'delete-sms-notifications',
+            'schedule': crontab(hour=4, minute=15),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-email-notifications': {
+            'task': 'delete-email-notifications',
+            'schedule': crontab(hour=4, minute=30),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-letter-notifications': {
+            'task': 'delete-letter-notifications',
+            'schedule': crontab(hour=4, minute=45),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(hour=1, minute=40),
+            'options': {'queue': QueueNames.PERIODIC}
+        },

-    # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
-    if os.environ.get('CELERYD_PREFETCH_MULTIPLIER'):
-        CELERY['worker_prefetch_multiplier'] = os.environ.get('CELERYD_PREFETCH_MULTIPLIER')
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'remove_sms_email_jobs': {
+            'task': 'remove_sms_email_jobs',
+            'schedule': crontab(hour=4, minute=0),
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'remove_letter_jobs': {
+            'task': 'remove_letter_jobs',
+            'schedule': crontab(hour=4, minute=20),
+            # since we mark jobs as archived
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+            'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+            'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-letter-notifications-still-sending': {
+            'task': 'raise-alert-if-letter-notifications-still-sending',
+            'schedule': crontab(hour=15, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-to-be-sent': {
+            'task': 'collate-letter-pdfs-to-be-sent',
+            'schedule': crontab(hour=17, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-no-letter-ack-file': {
+            'task': 'raise-alert-if-no-letter-ack-file',
+            'schedule': crontab(hour=23, minute=00),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+    }
+    CELERY_QUEUES = []

    FROM_NUMBER = 'development'

-    STATSD_HOST = os.environ.get('STATSD_HOST')
+    STATSD_HOST = os.getenv('STATSD_HOST')
    STATSD_PORT = 8125
    STATSD_ENABLED = bool(STATSD_HOST)

@@ -374,10 +328,13 @@ class Config(object):
    FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
    MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
    MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
-    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
-    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')

-    HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
+    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
+    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
+
+    # Format is as follows:
+    # {"dataset_1": "token_1", ...}
+    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))

    TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
    TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
@@ -389,16 +346,7 @@ class Config(object):
    MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
    FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")

-    AWS_REGION = 'us-west-2'
-
-    CBC_PROXY_ENABLED = True
-    CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
-    CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '')
-
-    ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE}
-
-    # as defined in api db migration 0331_add_broadcast_org.py
-    BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
+    AWS_REGION = 'eu-west-1'


 ######################
@@ -409,10 +357,7 @@ class Development(Config):
    DEBUG = True
    SQLALCHEMY_ECHO = False

-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
-
-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
+    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
    TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
    DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
    LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
@@ -421,11 +366,7 @@ class Development(Config):
    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
    LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'

-    # INTERNAL_CLIENT_API_KEYS = {
-    #     Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
-    #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
-    # }
-
+    API_INTERNAL_SECRETS = ['dev-notify-secret-key']
    SECRET_KEY = 'dev-notify-secret-key'
    DANGEROUS_SALT = 'dev-notify-salt'

@@ -434,21 +375,21 @@ class Development(Config):

    NOTIFY_ENVIRONMENT = 'development'
    NOTIFY_LOG_PATH = 'application.log'
-    NOTIFY_EMAIL_DOMAIN = "dispostable.com"
+    NOTIFICATION_QUEUE_PREFIX = 'development'
+    NOTIFY_EMAIL_DOMAIN = "notify.tools"

-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
-    REDIS_URL = os.environ.get('REDIS_URL')
+    SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
+    REDIS_URL = 'redis://localhost:6379/0'

-    ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
+    ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'

-    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
-
-    API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011')
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )

+    API_HOST_NAME = "http://localhost:6011"
    API_RATE_LIMIT_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
-
-    CBC_PROXY_ENABLED = False


 class Test(Development):
@@ -457,15 +398,7 @@ class Test(Development):
    NOTIFY_ENVIRONMENT = 'test'
    TESTING = True

-    HIGH_VOLUME_SERVICE = [
-        '941b6f9a-50d7-4742-8d50-f365ca74bf27',
-        '63f95b86-2d19-4497-b8b2-ccf25457df4e',
-        '7e5950cb-9954-41f5-8376-962b8c8555cf',
-        '10d1b9c9-0072-4fa9-ae1c-595e333841da',
-    ]
-
    CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
    TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
    DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
    LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
@@ -474,16 +407,18 @@ class Test(Development):
    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
    LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'

-    # this is overriden in CI
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api')
+    # this is overriden in jenkins and on cloudfoundry
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')

-    CELERY = {
-        **Config.CELERY,
-        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
-    }
+    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'

    ANTIVIRUS_ENABLED = True

+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
    API_RATE_LIMIT_ENABLED = True
    API_HOST_NAME = "http://localhost:6011"

@@ -494,15 +429,11 @@ class Test(Development):
    MMG_URL = 'https://example.com/mmg'
    FIRETEXT_URL = 'https://example.com/firetext'

-    CBC_PROXY_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
-

 class Preview(Config):
    NOTIFY_EMAIL_DOMAIN = 'notify.works'
    NOTIFY_ENVIRONMENT = 'preview'
    CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'preview-contact-list'
    TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
    LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
@@ -519,7 +450,6 @@ class Staging(Config):
    NOTIFY_EMAIL_DOMAIN = 'staging-notify.works'
    NOTIFY_ENVIRONMENT = 'staging'
    CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'staging-contact-list'
    TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
    DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
    LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
@@ -530,34 +460,26 @@ class Staging(Config):
    FROM_NUMBER = 'stage'
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = True
+    REDIS_ENABLED = True


 class Live(Config):
-    NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN')
+    NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
    NOTIFY_ENVIRONMENT = 'live'
-    # buckets
-    CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox
-    CONTACT_LIST_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox
-    # TODO: verify below buckets only used for letters
-    TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox
-    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox
-    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox
-    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox
-    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
-    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
-    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
-    
-    FROM_NUMBER = 'US Notify'
+    CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
+    FROM_NUMBER = 'GOVUK'
+    PERFORMANCE_PLATFORM_ENABLED = True
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = True
-    SES_STUB_URL = None
-    CRONITOR_ENABLED = True
-    
-    # DEBUG = True
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')

-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log')
-    REDIS_URL = os.environ.get('REDIS_URL')
+    CRONITOR_ENABLED = True


 class CloudFoundryConfig(Config):
@@ -569,7 +491,6 @@ class Sandbox(CloudFoundryConfig):
    NOTIFY_EMAIL_DOMAIN = 'notify.works'
    NOTIFY_ENVIRONMENT = 'sandbox'
    CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list'
    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
@@ -577,6 +498,7 @@ class Sandbox(CloudFoundryConfig):
    LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
    INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
    FROM_NUMBER = 'sandbox'
+    REDIS_ENABLED = False


 configs = {
--- a/app/cronitor.py
+++ b/app/cronitor.py
@@ -1,19 +1,15 @@
-from functools import wraps
-
 import requests
+from functools import wraps
 from flask import current_app


 def cronitor(task_name):
+    # check if task_name is in config
    def decorator(func):
        def ping_cronitor(command):
            if not current_app.config['CRONITOR_ENABLED']:
                return

-            # it's useful to have a log that a periodic task has started in case it
-            # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
-
            task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
            if not task_slug:
                current_app.logger.error(
@@ -42,11 +38,13 @@ def cronitor(task_name):
        @wraps(func)
        def inner_decorator(*args, **kwargs):
            ping_cronitor('run')
-            status = 'fail'
            try:
                ret = func(*args, **kwargs)
                status = 'complete'
                return ret
+            except Exception:
+                status = 'fail'
+                raise
            finally:
                ping_cronitor(status)

--- a/app/dao/init.py
+++ b/app/dao/init.py
@@ -1,5 +1,4 @@
 from sqlalchemy.exc import SQLAlchemyError
-
 from app import db


--- a/app/dao/annual_billing_dao.py
+++ b/app/dao/annual_billing_dao.py
@@ -1,12 +1,10 @@
-from flask import current_app
-
 from app import db
-from app.dao.dao_utils import autocommit
-from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.dao_utils import transactional
 from app.models import AnnualBilling
+from app.dao.date_util import get_current_financial_year_start_year


-@autocommit
+@transactional
 def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
    result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)

@@ -25,7 +23,7 @@ def dao_get_annual_billing(service_id):
    ).order_by(AnnualBilling.financial_year_start).all()


-@autocommit
+@transactional
 def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
    AnnualBilling.query.filter(
        AnnualBilling.service_id == service_id,
@@ -51,67 +49,3 @@ def dao_get_all_free_sms_fragment_limit(service_id):
    return AnnualBilling.query.filter_by(
        service_id=service_id,
    ).order_by(AnnualBilling.financial_year_start).all()
-
-
-def set_default_free_allowance_for_service(service, year_start=None):
-    default_free_sms_fragment_limits = {
-        'central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'nhs_local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_gp': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'emergency_service': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'school_or_college': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'other': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-    }
-    if not year_start:
-        year_start = get_current_financial_year_start_year()
-    # handle cases where the year is less than 2020 or greater than 2021
-    if year_start < 2020:
-        year_start = 2020
-    if year_start > 2022:
-        year_start = 2022
-    if service.organisation_type:
-        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
-    else:
-        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
-                                f"{default_free_sms_fragment_limits['other'][year_start]}")
-        free_allowance = default_free_sms_fragment_limits['other'][year_start]
-
-    return dao_create_or_update_annual_billing_for_year(
-        service.id,
-        free_allowance,
-        year_start
-    )
--- a/app/dao/api_key_dao.py
+++ b/app/dao/api_key_dao.py
@@ -1,14 +1,18 @@
 import uuid
 from datetime import datetime, timedelta

-from sqlalchemy import func, or_
-
 from app import db
-from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey

+from app.dao.dao_utils import (
+    transactional,
+    version_class
+)

-@autocommit
+from sqlalchemy import or_, func
+
+
+@transactional
@version_class(ApiKey)
 def save_model_api_key(api_key):
    if not api_key.id:
@@ -17,7 +21,7 @@ def save_model_api_key(api_key):
    db.session.add(api_key)


-@autocommit
+@transactional
@version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
    api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
--- a/app/dao/broadcast_message_dao.py
+++ b/app/dao/broadcast_message_dao.py
@@ -1,105 +0,0 @@
-import uuid
-from datetime import datetime
-
-from sqlalchemy import desc
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    BroadcastEvent,
-    BroadcastMessage,
-    BroadcastProvider,
-    BroadcastProviderMessage,
-    BroadcastProviderMessageNumber,
-    BroadcastProviderMessageStatus,
-    BroadcastStatusType,
-    ServiceBroadcastSettings,
-)
-
-
-def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.id == broadcast_message_id,
-        BroadcastMessage.service_id == service_id
-    ).one()
-
-
-def dao_get_broadcast_message_by_references_and_service_id(references_to_original_broadcast, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.status.in_((
-            BroadcastStatusType.PENDING_APPROVAL,
-            BroadcastStatusType.BROADCASTING,
-        )),
-        BroadcastMessage.reference.in_(references_to_original_broadcast),
-        BroadcastMessage.service_id == service_id
-    ).one()
-
-
-def dao_get_broadcast_event_by_id(broadcast_event_id):
-    return BroadcastEvent.query.filter(BroadcastEvent.id == broadcast_event_id).one()
-
-
-def dao_get_broadcast_messages_for_service(service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.service_id == service_id
-    ).order_by(BroadcastMessage.created_at)
-
-
-def dao_get_all_broadcast_messages():
-    return db.session.query(
-        BroadcastMessage.id,
-        BroadcastMessage.reference,
-        ServiceBroadcastSettings.channel,
-        BroadcastMessage.content,
-        BroadcastMessage.areas,
-        BroadcastMessage.status,
-        BroadcastMessage.starts_at,
-        BroadcastMessage.finishes_at,
-        BroadcastMessage.approved_at,
-        BroadcastMessage.cancelled_at,
-    ).join(
-        ServiceBroadcastSettings, ServiceBroadcastSettings.service_id == BroadcastMessage.service_id
-    ).filter(
-        BroadcastMessage.starts_at >= datetime(2021, 5, 25, 0, 0, 0),
-        BroadcastMessage.stubbed == False,  # noqa
-        BroadcastMessage.status.in_(BroadcastStatusType.LIVE_STATUSES)
-    ).order_by(desc(BroadcastMessage.starts_at)).all()
-
-
-def get_earlier_events_for_broadcast_event(broadcast_event_id):
-    """
-    This is used to build up the references list.
-    """
-    this_event = BroadcastEvent.query.get(broadcast_event_id)
-
-    return BroadcastEvent.query.filter(
-        BroadcastEvent.broadcast_message_id == this_event.broadcast_message_id,
-        BroadcastEvent.sent_at < this_event.sent_at
-    ).order_by(
-        BroadcastEvent.sent_at.asc()
-    ).all()
-
-
-@autocommit
-def create_broadcast_provider_message(broadcast_event, provider):
-    broadcast_provider_message_id = uuid.uuid4()
-    provider_message = BroadcastProviderMessage(
-        id=broadcast_provider_message_id,
-        broadcast_event=broadcast_event,
-        provider=provider,
-        status=BroadcastProviderMessageStatus.SENDING,
-    )
-    db.session.add(provider_message)
-    db.session.commit()
-    provider_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        provider_message_number = BroadcastProviderMessageNumber(
-            broadcast_provider_message_id=broadcast_provider_message_id)
-        db.session.add(provider_message_number)
-        db.session.commit()
-    return provider_message
-
-
-@autocommit
-def update_broadcast_provider_message_status(broadcast_provider_message, *, status):
-    broadcast_provider_message.status = status
--- a/app/dao/broadcast_service_dao.py
+++ b/app/dao/broadcast_service_dao.py
@@ -1,100 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    BROADCAST_TYPE,
-    EMAIL_AUTH_TYPE,
-    INVITE_PENDING,
-    VIEW_ACTIVITY,
-    ApiKey,
-    InvitedUser,
-    Organisation,
-    Permission,
-    Service,
-    ServiceBroadcastSettings,
-    ServicePermission,
-)
-
-
-@autocommit
-@version_class(Service)
-def set_broadcast_service_type(service, service_mode, broadcast_channel, provider_restriction):
-    insert_or_update_service_broadcast_settings(
-        service, channel=broadcast_channel, provider_restriction=provider_restriction
-    )
-
-    # Remove all permissions and add broadcast permission
-    if not service.has_permission(BROADCAST_TYPE):
-        service_permission = ServicePermission(service_id=service.id, permission=BROADCAST_TYPE)
-        db.session.add(service_permission)
-
-    ServicePermission.query.filter(
-        ServicePermission.service_id == service.id,
-        ServicePermission.permission != BROADCAST_TYPE,
-        # Email auth is an exception to the other service permissions (which relate to what type
-        # of notifications a service can send) where a broadcast service is allowed to have the
-        # email auth permission (but doesn't have to)
-        ServicePermission.permission != EMAIL_AUTH_TYPE
-    ).delete()
-
-    # Refresh the service object as it has references to the service permissions but we don't yet
-    # want to commit the permission changes incase all of this needs to rollback
-    db.session.refresh(service)
-
-    # Set service count as live false always
-    service.count_as_live = False
-
-    # Set service into training mode or live mode
-    if service_mode == "live":
-        if service.restricted:
-            # Only update the go live at timestamp if this if moving from training mode
-            # to live mode, not if it's moving from one type of live mode service to another
-            service.go_live_at = datetime.utcnow()
-        service.restricted = False
-    else:
-        service.restricted = True
-        service.go_live_at = None
-
-    # Remove all user permissions apart from view_activity for the service users and invited users
-    Permission.query.filter(
-        Permission.service_id == service.id,
-        Permission.permission != VIEW_ACTIVITY
-    ).delete()
-    InvitedUser.query.filter_by(
-        service_id=service.id,
-        status=INVITE_PENDING
-    ).update({'permissions': VIEW_ACTIVITY})
-
-    # Revoke any API keys to avoid a regular API key being used to send alerts
-    ApiKey.query.filter_by(
-        service_id=service.id,
-        expiry_date=None,
-    ).update({
-        ApiKey.expiry_date: datetime.utcnow()
-    })
-
-    # Add service to organisation
-    organisation = Organisation.query.filter_by(
-        id=current_app.config['BROADCAST_ORGANISATION_ID']
-    ).one()
-    service.organisation_id = organisation.id
-    service.organisation_type = organisation.organisation_type
-    service.crown = organisation.crown
-
-    db.session.add(service)
-
-
-def insert_or_update_service_broadcast_settings(service, channel, provider_restriction="all"):
-    if not service.service_broadcast_settings:
-        settings = ServiceBroadcastSettings()
-        settings.service = service
-        settings.channel = channel
-        settings.provider = provider_restriction
-        db.session.add(settings)
-    else:
-        service.service_broadcast_settings.channel = channel
-        service.service_broadcast_settings.provider = provider_restriction
-        db.session.add(service.service_broadcast_settings)
--- a/app/dao/complaint_dao.py
+++ b/app/dao/complaint_dao.py
@@ -4,12 +4,12 @@ from flask import current_app
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import Complaint
 from app.utils import get_london_midnight_in_utc


-@autocommit
+@transactional
 def save_complaint(complaint):
    db.session.add(complaint)

--- a/app/dao/daily_sorted_letter_dao.py
+++ b/app/dao/daily_sorted_letter_dao.py
@@ -3,7 +3,7 @@ from datetime import datetime
 from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import DailySortedLetter


@@ -13,7 +13,7 @@ def dao_get_daily_sorted_letter_by_billing_day(billing_day):
    ).first()


-@autocommit
+@transactional
 def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
    '''
    This uses the Postgres upsert to avoid race conditions when two threads try and insert
--- a/app/dao/dao_utils.py
+++ b/app/dao/dao_utils.py
@@ -1,20 +1,16 @@
 import itertools
-from contextlib import contextmanager
 from functools import wraps

 from app import db
 from app.history_meta import create_history


-def autocommit(func):
+def transactional(func):
    @wraps(func)
    def commit_or_rollback(*args, **kwargs):
        try:
            res = func(*args, **kwargs)
-
-            if not db.session.registry().transaction.nested:
-                db.session.commit()
-
+            db.session.commit()
            return res
        except Exception:
            db.session.rollback()
@@ -22,20 +18,6 @@ def autocommit(func):
    return commit_or_rollback


-@contextmanager
-def transaction():
-    try:
-        db.session.begin_nested()
-        yield
-        db.session.commit()
-
-        if not db.session.registry().transaction.nested:
-            db.session.commit()
-    except Exception:
-        db.session.rollback()
-        raise
-
-
 class VersionOptions():

    def __init__(self, model_class, history_class=None, must_write_history=True):
@@ -91,9 +73,3 @@ def version_class(*version_options):

 def dao_rollback():
    db.session.rollback()
-
-
-@autocommit
-def dao_save_object(obj):
-    # add/update object in db
-    db.session.add(obj)
--- a/app/dao/date_util.py
+++ b/app/dao/date_util.py
@@ -1,7 +1,7 @@
-from datetime import date, datetime, time, timedelta
+from datetime import datetime, timedelta, date, time

+from notifications_utils.timezones import convert_bst_to_utc
 import pytz
-from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst


 def get_months_for_financial_year(year):
@@ -22,15 +22,6 @@ def get_financial_year(year):
    return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)


-def get_financial_year_dates(year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    return (
-        convert_utc_to_bst(year_start_datetime).date(),
-        convert_utc_to_bst(year_end_datetime).date()
-    )
-
-
 def get_current_financial_year():
    now = datetime.utcnow()
    current_month = int(now.strftime('%-m'))
--- a/app/dao/email_branding_dao.py
+++ b/app/dao/email_branding_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import EmailBranding


@@ -15,12 +15,12 @@ def dao_get_email_branding_by_name(email_branding_name):
    return EmailBranding.query.filter_by(name=email_branding_name).first()


-@autocommit
+@transactional
 def dao_create_email_branding(email_branding):
    db.session.add(email_branding)


-@autocommit
+@transactional
 def dao_update_email_branding(email_branding, **kwargs):
    for key, value in kwargs.items():
        setattr(email_branding, key, value or None)
--- a/app/dao/fact_billing_dao.py
+++ b/app/dao/fact_billing_dao.py
--- a/app/dao/fact_notification_status_dao.py
+++ b/app/dao/fact_notification_status_dao.py
@@ -1,91 +1,114 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time

-from sqlalchemy import Date, case, func
+from flask import current_app
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.timezones import convert_bst_to_utc
+from sqlalchemy import case, func, Date
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import extract, literal
+from sqlalchemy.sql.expression import literal, extract
 from sqlalchemy.types import DateTime, Integer

 from app import db
-from app.dao.dao_utils import autocommit
 from app.models import (
-    KEY_TYPE_NORMAL,
-    KEY_TYPE_TEAM,
+    FactNotificationStatus,
    KEY_TYPE_TEST,
+    Notification,
    NOTIFICATION_CANCELLED,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_FAILED,
-    NOTIFICATION_PENDING,
-    NOTIFICATION_PERMANENT_FAILURE,
    NOTIFICATION_SENDING,
    NOTIFICATION_SENT,
    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
-    FactNotificationStatus,
-    Notification,
-    NotificationAllTimeView,
+    NOTIFICATION_PERMANENT_FAILURE,
    Service,
    Template,
 )
+from app.dao.dao_utils import transactional
 from app.utils import (
    get_london_midnight_in_utc,
-    get_london_month_from_utc_column,
    midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
 )


-@autocommit
-def update_fact_notification_status(process_day, notification_type, service_id):
-    start_date = get_london_midnight_in_utc(process_day)
-    end_date = get_london_midnight_in_utc(process_day + timedelta(days=1))
+@statsd(namespace="dao")
+def fetch_notification_status_for_day(process_day, notification_type):
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))

-    # delete any existing rows in case some no longer exist e.g. if all messages are sent
-    FactNotificationStatus.query.filter(
-        FactNotificationStatus.bst_date == process_day,
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.service_id == service_id,
-    ).delete()
+    current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))

+    all_data_for_process_day = []
+    services = Service.query.all()
+    # for each service query notifications or notification_history for the day, depending on their data retention
+    for service in services:
+        table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
+
+        data_for_service_and_type = query_for_fact_status_data(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+            notification_type=notification_type,
+            service_id=service.id
+        )
+
+        all_data_for_process_day += data_for_service_and_type
+
+    return all_data_for_process_day
+
+
+@statsd(namespace="dao")
+def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
    query = db.session.query(
-        literal(process_day).label("process_day"),
-        NotificationAllTimeView.template_id,
-        literal(service_id).label("service_id"),
-        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
-        literal(notification_type).label("notification_type"),
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status,
+        table.template_id,
+        table.service_id,
+        func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        table.key_type,
+        table.status,
        func.count().label('notification_count')
    ).filter(
-        NotificationAllTimeView.created_at >= start_date,
-        NotificationAllTimeView.created_at < end_date,
-        NotificationAllTimeView.notification_type == notification_type,
-        NotificationAllTimeView.service_id == service_id,
-        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        table.created_at >= start_date,
+        table.created_at < end_date,
+        table.notification_type == notification_type,
+        table.service_id == service_id,
+        table.key_type != KEY_TYPE_TEST
    ).group_by(
-        NotificationAllTimeView.template_id,
-        NotificationAllTimeView.template_id,
+        table.template_id,
+        table.service_id,
        'job_id',
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status
+        table.key_type,
+        table.status
    )
+    return query.all()

-    db.session.connection().execute(
-        insert(FactNotificationStatus.__table__).from_select(
-            [
-                FactNotificationStatus.bst_date,
-                FactNotificationStatus.template_id,
-                FactNotificationStatus.service_id,
-                FactNotificationStatus.job_id,
-                FactNotificationStatus.notification_type,
-                FactNotificationStatus.key_type,
-                FactNotificationStatus.notification_status,
-                FactNotificationStatus.notification_count
-            ],
-            query
+
+@statsd(namespace="dao")
+@transactional
+def update_fact_notification_status(data, process_day, notification_type):
+    table = FactNotificationStatus.__table__
+    FactNotificationStatus.query.filter(
+        FactNotificationStatus.bst_date == process_day,
+        FactNotificationStatus.notification_type == notification_type
+    ).delete()
+
+    for row in data:
+        stmt = insert(table).values(
+            bst_date=process_day,
+            template_id=row.template_id,
+            service_id=row.service_id,
+            job_id=row.job_id,
+            notification_type=notification_type,
+            key_type=row.key_type,
+            notification_status=row.status,
+            notification_count=row.notification_count,
        )
-    )
+        db.session.connection().execute(stmt)


+@statsd(namespace="dao")
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
    return db.session.query(
        func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
@@ -104,6 +127,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
    ).all()


+@statsd(namespace="dao")
 def fetch_notification_status_for_service_for_day(bst_day, service_id):
    return db.session.query(
        # return current month as a datetime so the data has the same shape as the ft_notification_status query
@@ -122,6 +146,7 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
    ).all()


+@statsd(namespace="dao")
 def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
    start_date = midnight_n_days_ago(limit_days)
    now = datetime.utcnow()
@@ -174,6 +199,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
    ).all()


+@statsd(namespace="dao")
 def fetch_notification_status_totals_for_all_services(start_date, end_date):
    stats = db.session.query(
        FactNotificationStatus.notification_type.label('notification_type'),
@@ -222,6 +248,7 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
    return query.all()


+@statsd(namespace="dao")
 def fetch_notification_statuses_for_job(job_id):
    return db.session.query(
        FactNotificationStatus.notification_status.label('status'),
@@ -233,6 +260,7 @@ def fetch_notification_statuses_for_job(job_id):
    ).all()


+@statsd(namespace="dao")
 def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True):
    stats = db.session.query(
        FactNotificationStatus.service_id.label('service_id'),
@@ -327,6 +355,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
    return query.all()


+@statsd(namespace="dao")
 def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
    # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service
    stats = db.session.query(
@@ -411,39 +440,20 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
    return query.all()


-def get_total_notifications_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('emails'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('sms'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('letters'),
+@statsd(namespace="dao")
+def get_total_sent_notifications_for_day_and_type(day, notification_type):
+    result = db.session.query(
+        func.sum(FactNotificationStatus.notification_count).label('count')
    ).filter(
+        FactNotificationStatus.notification_type == notification_type,
        FactNotificationStatus.key_type != KEY_TYPE_TEST,
-    ).group_by(
-        FactNotificationStatus.bst_date
-    ).order_by(
-        FactNotificationStatus.bst_date
-    )
-    if start_date and end_date:
-        query = query.filter(
-            FactNotificationStatus.bst_date >= start_date,
-            FactNotificationStatus.bst_date <= end_date
-        )
-    return query.all()
+        FactNotificationStatus.bst_date == day,
+    ).scalar()
+
+    return result or 0


+@statsd(namespace="dao")
 def fetch_monthly_notification_statuses_per_service(start_date, end_date):
    return db.session.query(
        func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'),
@@ -452,7 +462,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
        FactNotificationStatus.notification_type,
        func.sum(case(
            [
-                (FactNotificationStatus.notification_status.in_([NOTIFICATION_SENDING, NOTIFICATION_PENDING]),
+                (FactNotificationStatus.notification_status == NOTIFICATION_SENDING,
                 FactNotificationStatus.notification_count)
            ],
            else_=0)).label('count_sending'),
--- a/app/dao/fact_processing_time_dao.py
+++ b/app/dao/fact_processing_time_dao.py
@@ -1,53 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import case
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import FactProcessingTime
-
-
-@autocommit
-def insert_update_processing_time(processing_time):
-    '''
-    This uses the Postgres upsert to avoid race conditions when two threads try and insert
-    at the same row. The excluded object refers to values that we tried to insert but were
-    rejected.
-    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    '''
-    table = FactProcessingTime.__table__
-    stmt = insert(table).values(
-        bst_date=processing_time.bst_date,
-        messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs
-    )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.bst_date],
-        set_={
-            'messages_total': stmt.excluded.messages_total,
-            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
-            'updated_at': datetime.utcnow()
-        }
-    )
-    db.session.connection().execute(stmt)
-
-
-def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactProcessingTime.bst_date.cast(db.Text).label("date"),
-        FactProcessingTime.messages_total,
-        FactProcessingTime.messages_within_10_secs,
-        case([
-            (
-                FactProcessingTime.messages_total > 0,
-                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
-            ),
-            (FactProcessingTime.messages_total == 0, 100.0)
-        ]).label("percentage")
-    ).filter(
-        FactProcessingTime.bst_date >= start_date,
-        FactProcessingTime.bst_date <= end_date
-    ).order_by(FactProcessingTime.bst_date)
-
-    return query.all()
--- a/app/dao/inbound_numbers_dao.py
+++ b/app/dao/inbound_numbers_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import InboundNumber


@@ -19,13 +19,13 @@ def dao_get_inbound_number(inbound_number_id):
    return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()


-@autocommit
+@transactional
 def dao_set_inbound_number_to_service(service_id, inbound_number):
    inbound_number.service_id = service_id
    db.session.add(inbound_number)


-@autocommit
+@transactional
 def dao_set_inbound_number_active_flag(service_id, active):
    inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
    inbound_number.active = active
@@ -33,7 +33,7 @@ def dao_set_inbound_number_active_flag(service_id, active):
    db.session.add(inbound_number)


-@autocommit
+@transactional
 def dao_allocate_number_for_service(service_id, inbound_number_id):
    updated = InboundNumber.query.filter_by(
        id=inbound_number_id,
--- a/app/dao/inbound_sms_dao.py
+++ b/app/dao/inbound_sms_dao.py
@@ -1,21 +1,16 @@
 from flask import current_app
-from sqlalchemy import and_, desc
-from sqlalchemy.dialects.postgresql import insert
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import desc, and_
 from sqlalchemy.orm import aliased
+from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    InboundSms,
-    InboundSmsHistory,
-    Service,
-    ServiceDataRetention,
-)
+from app.dao.dao_utils import transactional
+from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
 from app.utils import midnight_n_days_ago


-@autocommit
+@transactional
 def dao_create_inbound_sms(inbound_sms):
    db.session.add(inbound_sms)

@@ -71,13 +66,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days):
 def _insert_inbound_sms_history(subquery, query_limit=10000):
    offset = 0
    inbound_sms_query = db.session.query(
-        InboundSms.id,
-        InboundSms.created_at,
-        InboundSms.service_id,
-        InboundSms.notify_number,
-        InboundSms.provider_date,
-        InboundSms.provider_reference,
-        InboundSms.provider
+        *[x.name for x in InboundSmsHistory.__table__.c]
    ).filter(InboundSms.id.in_(subquery))
    inbound_sms_count = inbound_sms_query.count()

@@ -119,7 +108,8 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
    return deleted


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def delete_inbound_sms_older_than_retention():
    current_app.logger.info('Deleting inbound sms for services with flexible data retention')

--- a/app/dao/invited_org_user_dao.py
+++ b/app/dao/invited_org_user_dao.py
@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedOrganisationUser


--- a/app/dao/invited_user_dao.py
+++ b/app/dao/invited_user_dao.py
@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedUser


@@ -9,7 +9,7 @@ def save_invited_user(invited_user):
    db.session.commit()


-def get_invited_user_by_service_and_id(service_id, invited_user_id):
+def get_invited_user(service_id, invited_user_id):
    return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()


--- a/app/dao/jobs_dao.py
+++ b/app/dao/jobs_dao.py
@@ -2,32 +2,37 @@ import uuid
 from datetime import datetime, timedelta

 from flask import current_app
-from notifications_utils.letter_timings import (
-    CANCELLABLE_JOB_LETTER_STATUSES,
-    letter_can_be_cancelled,
+from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import (
+    asc,
+    desc,
+    func,
+    and_
 )
-from sqlalchemy import and_, asc, desc, func

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.dao.templates_dao import dao_get_template_by_id
+from app.utils import midnight_n_days_ago
+
 from app.models import (
-    JOB_STATUS_CANCELLED,
+    Job,
    JOB_STATUS_FINISHED,
    JOB_STATUS_PENDING,
    JOB_STATUS_SCHEDULED,
    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    NOTIFICATION_CREATED,
-    FactNotificationStatus,
-    Job,
    Notification,
-    ServiceDataRetention,
    Template,
+    ServiceDataRetention,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_CANCELLED,
+    JOB_STATUS_CANCELLED,
+    FactNotificationStatus
 )
-from app.utils import midnight_n_days_ago


+@statsd(namespace="dao")
 def dao_get_notification_outcomes_for_job(service_id, job_id):
    notification_statuses = db.session.query(
        func.count(Notification.status).label('count'), Notification.status
@@ -53,15 +58,7 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
    return Job.query.filter_by(service_id=service_id, id=job_id).one()


-def dao_get_jobs_by_service_id(
-    service_id,
-    *,
-    limit_days=None,
-    page=1,
-    page_size=50,
-    statuses=None,
-    contact_list_id=None,
-):
+def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
    query_filter = [
        Job.service_id == service_id,
        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
@@ -73,26 +70,12 @@ def dao_get_jobs_by_service_id(
        query_filter.append(
            Job.job_status.in_(statuses)
        )
-    if contact_list_id is not None:
-        query_filter.append(Job.contact_list_id == contact_list_id)
    return Job.query \
        .filter(*query_filter) \
        .order_by(Job.processing_started.desc(), Job.created_at.desc()) \
        .paginate(page=page, per_page=page_size)


-def dao_get_scheduled_job_stats(
-    service_id,
-):
-    return db.session.query(
-        func.count(Job.id),
-        func.min(Job.scheduled_for),
-    ).filter(
-        Job.service_id == service_id,
-        Job.job_status == JOB_STATUS_SCHEDULED,
-    ).one()
-
-
 def dao_get_job_by_id(job_id):
    return Job.query.filter_by(id=job_id).one()

@@ -183,7 +166,7 @@ def dao_get_jobs_older_than_data_retention(notification_types):
    return jobs


-@autocommit
+@transactional
 def dao_cancel_letter_job(job):
    number_of_notifications_cancelled = Notification.query.filter(
        Notification.job_id == job.id
@@ -218,11 +201,12 @@ def can_letter_job_be_cancelled(job):


 def find_jobs_with_missing_rows():
-    # Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
-    # Using 20 minutes as a condition seems reasonable.
-    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=20)
+    # Jobs can be a maximum of 50,000 rows. It typically takes 5 minutes to create all those notifications.
+    # Using 10 minutes as a condition seems reasonable.
+    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=10)
    yesterday = datetime.utcnow() - timedelta(days=1)
    jobs_with_rows_missing = db.session.query(
+        func.count(Notification.id).label('actual_count'),
        Job
    ).filter(
        Job.job_status == JOB_STATUS_FINISHED,
--- a/app/dao/letter_branding_dao.py
+++ b/app/dao/letter_branding_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import LetterBranding


@@ -15,12 +15,12 @@ def dao_get_all_letter_branding():
    return LetterBranding.query.order_by(LetterBranding.name).all()


-@autocommit
+@transactional
 def dao_create_letter_branding(letter_branding):
    db.session.add(letter_branding)


-@autocommit
+@transactional
 def dao_update_letter_branding(letter_branding_id, **kwargs):
    letter_branding = LetterBranding.query.get(letter_branding_id)
    for key, value in kwargs.items():
--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -1,55 +1,62 @@
-from datetime import datetime, timedelta
+import functools
 from itertools import groupby
 from operator import attrgetter
+from datetime import (
+    datetime,
+    timedelta,
+)

 from botocore.exceptions import ClientError
 from flask import current_app
-from notifications_utils.international_billing_rates import (
-    INTERNATIONAL_BILLING_RATES,
-)
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
    validate_and_format_email_address,
+    InvalidEmailError,
+    try_validate_and_format_phone_number
 )
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
-from sqlalchemy import and_, asc, desc, func, or_, union
+from sqlalchemy import (desc, func, asc, and_, or_)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
+from sqlalchemy.dialects.postgresql import insert
 from werkzeug.datastructures import MultiDict

-from app import create_uuid, db, statsd_client
-from app.dao.dao_utils import autocommit
-from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
+from app import db, create_uuid
+from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
+from app.dao.dao_utils import transactional
+from app.errors import InvalidRequest
+from app.letters.utils import get_letter_pdf_filename
 from app.models import (
-    EMAIL_TYPE,
+    FactNotificationStatus,
+    Notification,
+    NotificationHistory,
+    ProviderDetails,
+    ScheduledNotification,
    KEY_TYPE_NORMAL,
    KEY_TYPE_TEST,
    LETTER_TYPE,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
    NOTIFICATION_PENDING,
    NOTIFICATION_PENDING_VIRUS_CHECK,
-    NOTIFICATION_PERMANENT_FAILURE,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
+    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENT,
    SMS_TYPE,
-    FactNotificationStatus,
-    Notification,
-    NotificationHistory,
-    ProviderDetails,
-)
-from app.utils import (
-    escape_special_characters,
-    get_london_midnight_in_utc,
-    midnight_n_days_ago,
+    EMAIL_TYPE,
+    ServiceDataRetention,
+    Service,
 )
+from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters


+@statsd(namespace="dao")
 def dao_get_last_date_template_was_used(template_id, service_id):
    last_date_from_notifications = db.session.query(
        functions.max(Notification.created_at)
@@ -72,7 +79,8 @@ def dao_get_last_date_template_was_used(template_id, service_id):
    return last_date


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_create_notification(notification):
    if not notification.id:
        # need to populate defaulted fields before we create the notification history object
@@ -83,22 +91,29 @@ def dao_create_notification(notification):
    db.session.add(notification)


+def _decide_permanent_temporary_failure(current_status, status):
+    # Firetext will send pending, then send either succes or fail.
+    # If we go from pending to delivered we need to set failure type as temporary-failure
+    if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+    return status
+
+
 def country_records_delivery(phone_prefix):
    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
    return dlr and dlr.lower() == 'yes'


-def _update_notification_status(notification, status, detailed_status_code=None):
-    # status = _decide_permanent_temporary_failure(
-    #     status=status, notification=notification, detailed_status_code=detailed_status_code
-    # )
-    # notification.status = status
-    # dao_update_notification(notification)
+def _update_notification_status(notification, status):
+    status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
+    notification.status = status
+    dao_update_notification(notification)
    return notification


-@autocommit
-def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
+@statsd(namespace="dao")
+@transactional
+def update_notification_status_by_id(notification_id, status, sent_by=None):
    notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()

    if not notification:
@@ -118,22 +133,18 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
        _duplicate_update_warning(notification, status)
        return None

-    if (
-        notification.notification_type == SMS_TYPE
-        and notification.international
-        and not country_records_delivery(notification.phone_prefix)
-    ):
+    if notification.international and not country_records_delivery(notification.phone_prefix):
        return None
    if not notification.sent_by and sent_by:
        notification.sent_by = sent_by
    return _update_notification_status(
        notification=notification,
-        status=status,
-        detailed_status_code=detailed_status_code
+        status=status
    )


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def update_notification_status_by_reference(reference, status):
    # this is used to update letters and emails
    notification = Notification.query.filter(Notification.reference == reference).first()
@@ -155,12 +166,19 @@ def update_notification_status_by_reference(reference, status):
    )


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_update_notification(notification):
    notification.updated_at = datetime.utcnow()
    db.session.add(notification)


+@statsd(namespace="dao")
+def get_notification_for_job(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
+@statsd(namespace="dao")
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
    if page_size is None:
        page_size = current_app.config['PAGE_SIZE']
@@ -172,10 +190,12 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
    )


+@statsd(namespace="dao")
 def dao_get_notification_count_for_job_id(*, job_id):
    return Notification.query.filter_by(job_id=job_id).count()


+@statsd(namespace="dao")
 def get_notification_with_personalisation(service_id, notification_id, key_type):
    filter_dict = {'service_id': service_id, 'id': notification_id}
    if key_type:
@@ -184,6 +204,7 @@ def get_notification_with_personalisation(service_id, notification_id, key_type)
    return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one()


+@statsd(namespace="dao")
 def get_notification_by_id(notification_id, service_id=None, _raise=False):
    filters = [Notification.id == notification_id]

@@ -195,6 +216,11 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
    return query.one() if _raise else query.first()


+def get_notifications(filter_dict=None):
+    return _filter_query(Notification.query, filter_dict=filter_dict)
+
+
+@statsd(namespace="dao")
 def get_notifications_for_service(
        service_id,
        filter_dict=None,
@@ -208,8 +234,7 @@ def get_notifications_for_service(
        include_from_test_key=False,
        older_than=None,
        client_reference=None,
-        include_one_off=True,
-        error_out=True
+        include_one_off=True
 ):
    if page_size is None:
        page_size = current_app.config['PAGE_SIZE']
@@ -248,8 +273,7 @@ def get_notifications_for_service(
    return query.order_by(desc(Notification.created_at)).paginate(
        page=page,
        per_page=page_size,
-        count=count_pages,
-        error_out=error_out,
+        count=count_pages
    )


@@ -273,117 +297,124 @@ def _filter_query(query, filter_dict=None):
    return query


-@autocommit
-def insert_notification_history_delete_notifications(
-    notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
-):
-    """
-    Delete up to 50,000 notifications that are past retention for a notification type and service.
+@statsd(namespace="dao")
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=10000):
+    current_app.logger.info(
+        'Deleting {} notifications for services with flexible data retention'.format(notification_type))

-
-    Steps are as follows:
-
-    Create a temporary notifications table
-    Populate that table with up to 50k notifications that are to be deleted. (Note: no specified order)
-    Insert everything in the temp table into notification history
-    Delete from notifications if notification id is in the temp table
-    Drop the temp table (automatically when the transaction commits)
-
-    Temporary tables are in a separate postgres schema, and only visible to the current session (db connection,
-    in a celery task there's one connection per thread.)
-    """
-    # Setting default query limit to 50,000 which take about 48 seconds on current table size
-    # 10, 000 took 11s and 100,000 took 1 min 30 seconds.
-    select_into_temp_table = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, postage, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
-        """
-    select_into_temp_table_for_letters = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, postage, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND notification_status NOT IN ('pending-virus-check', 'created', 'sending')
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
-        """
-    # Insert into NotificationHistory if the row already exists do nothing.
-    insert_query = """
-        insert into notification_history
-         SELECT * from NOTIFICATION_ARCHIVE
-          ON CONFLICT ON CONSTRAINT notification_history_pkey
-          DO NOTHING
-    """
-    delete_query = """
-        DELETE FROM notifications
-        where id in (select id from NOTIFICATION_ARCHIVE)
-    """
-    input_params = {
-        "service_id": service_id,
-        "notification_type": notification_type,
-        "timestamp_to_delete_backwards_from": timestamp_to_delete_backwards_from,
-        "qry_limit": qry_limit
-    }
-
-    select_to_use = select_into_temp_table_for_letters if notification_type == 'letter' else select_into_temp_table
-    db.session.execute(select_to_use, input_params)
-
-    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
-
-    db.session.execute(insert_query)
-
-    db.session.execute(delete_query)
-
-    return result
-
-
-def move_notifications_to_notification_history(
-    notification_type,
-    service_id,
-    timestamp_to_delete_backwards_from,
-    qry_limit=50000
-):
+    flexible_data_retention = ServiceDataRetention.query.filter(
+        ServiceDataRetention.notification_type == notification_type
+    ).all()
    deleted = 0
-    if notification_type == LETTER_TYPE:
-        _delete_letters_from_s3(
-            notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit
-        )
-    delete_count_per_call = 1
-    while delete_count_per_call > 0:
-        delete_count_per_call = insert_notification_history_delete_notifications(
-            notification_type=notification_type,
-            service_id=service_id,
-            timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
-            qry_limit=qry_limit
-        )
-        deleted += delete_count_per_call
+    for f in flexible_data_retention:
+        days_of_retention = get_london_midnight_in_utc(
+            convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)

-    # Deleting test Notifications, test notifications are not persisted to NotificationHistory
-    Notification.query.filter(
-        Notification.notification_type == notification_type,
-        Notification.service_id == service_id,
-        Notification.created_at < timestamp_to_delete_backwards_from,
-        Notification.key_type == KEY_TYPE_TEST
-    ).delete(synchronize_session=False)
-    db.session.commit()
+        if notification_type == LETTER_TYPE:
+            _delete_letters_from_s3(
+                notification_type, f.service_id, days_of_retention, qry_limit
+            )
+
+        insert_update_notification_history(notification_type, days_of_retention, f.service_id)
+
+        current_app.logger.info(
+            "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
+        deleted += _delete_notifications(notification_type, days_of_retention, f.service_id, qry_limit)
+
+    current_app.logger.info(
+        'Deleting {} notifications for services without flexible data retention'.format(notification_type))
+
+    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
+    services_with_data_retention = [x.service_id for x in flexible_data_retention]
+    service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
+
+    for service_id in service_ids_to_purge:
+        if notification_type == LETTER_TYPE:
+            _delete_letters_from_s3(
+                notification_type, service_id, seven_days_ago, qry_limit
+            )
+        insert_update_notification_history(notification_type, seven_days_ago, service_id)
+        deleted += _delete_notifications(notification_type, seven_days_ago, service_id, qry_limit)
+
+    current_app.logger.info('Finished deleting {} notifications'.format(notification_type))

    return deleted


+def _delete_notifications(notification_type, date_to_delete_from, service_id, query_limit):
+    subquery = db.session.query(
+        Notification.id
+    ).join(NotificationHistory, NotificationHistory.id == Notification.id).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+    ).limit(query_limit).subquery()
+
+    deleted = _delete_for_query(subquery)
+
+    subquery_for_test_keys = db.session.query(
+        Notification.id
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+        Notification.key_type == KEY_TYPE_TEST
+    ).limit(query_limit).subquery()
+
+    deleted += _delete_for_query(subquery_for_test_keys)
+
+    return deleted
+
+
+def _delete_for_query(subquery):
+    number_deleted = db.session.query(Notification).filter(
+        Notification.id.in_(subquery)).delete(synchronize_session='fetch')
+    deleted = number_deleted
+    db.session.commit()
+    while number_deleted > 0:
+        number_deleted = db.session.query(Notification).filter(
+            Notification.id.in_(subquery)).delete(synchronize_session='fetch')
+        deleted += number_deleted
+        db.session.commit()
+    return deleted
+
+
+def insert_update_notification_history(notification_type, date_to_delete_from, service_id, query_limit=10000):
+    offset = 0
+
+    notification_query = db.session.query(
+        *[x.name for x in NotificationHistory.__table__.c]
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+        Notification.key_type != KEY_TYPE_TEST
+    )
+    notifications_count = notification_query.count()
+
+    while offset < notifications_count:
+        stmt = insert(NotificationHistory).from_select(
+            NotificationHistory.__table__.c,
+            notification_query.limit(query_limit).offset(offset)
+        )
+
+        stmt = stmt.on_conflict_do_update(
+            constraint="notification_history_pkey",
+            set_={
+                "notification_status": stmt.excluded.status,
+                "reference": stmt.excluded.reference,
+                "billable_units": stmt.excluded.billable_units,
+                "updated_at": stmt.excluded.updated_at,
+                "sent_at": stmt.excluded.sent_at,
+                "sent_by": stmt.excluded.sent_by
+            }
+        )
+        db.session.connection().execute(stmt)
+        db.session.commit()
+
+        offset += query_limit
+
+
 def _delete_letters_from_s3(
        notification_type, service_id, date_to_delete_from, query_limit
 ):
@@ -392,55 +423,78 @@ def _delete_letters_from_s3(
    ).filter(
        Notification.notification_type == notification_type,
        Notification.created_at < date_to_delete_from,
-        Notification.service_id == service_id,
-        # although letters in non completed statuses do have PDFs in s3, they do not exist in the
-        # production-letters-pdf bucket as they never made it that far so we do not try and delete
-        # them from it
-        Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)
+        Notification.service_id == service_id
    ).limit(query_limit).all()
    for letter in letters_to_delete_from_s3:
-        try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
-            letter_pdf.delete()
-        except ClientError:
-            current_app.logger.exception(
-                "Error deleting S3 object for letter: {}".format(letter.id))
-        except LetterPDFNotFound:
-            current_app.logger.warning(
-                "No S3 object to delete for letter: {}".format(letter.id))
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+        # I don't think we need this anymore, we should update the query to get letters sent 7 days ago
+        if letter.sent_at:
+            prefix = get_letter_pdf_filename(reference=letter.reference,
+                                             crown=letter.service.crown,
+                                             sending_date=letter.created_at,
+                                             dont_use_sending_date=letter.key_type == KEY_TYPE_TEST,
+                                             postage=letter.postage)
+            s3_objects = get_s3_bucket_objects(bucket_name=bucket_name, subfolder=prefix)
+            for s3_object in s3_objects:
+                try:
+                    remove_s3_object(bucket_name, s3_object['Key'])
+                except ClientError:
+                    current_app.logger.exception(
+                        "Could not delete S3 object with filename: {}".format(s3_object['Key']))


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_delete_notifications_by_id(notification_id):
    db.session.query(Notification).filter(
        Notification.id == notification_id
    ).delete(synchronize_session='fetch')


-def dao_timeout_notifications(cutoff_time, limit=100000):
-    """
-    Set email and SMS notifications (only) to "temporary-failure" status
-    if they're still sending from before the specified cutoff_time.
-    """
-    updated_at = datetime.utcnow()
-    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
-    new_status = NOTIFICATION_TEMPORARY_FAILURE
-
+def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
    notifications = Notification.query.filter(
-        Notification.created_at < cutoff_time,
+        Notification.created_at < timeout_start,
        Notification.status.in_(current_statuses),
-        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
-    ).limit(limit).all()
-
+        Notification.notification_type != LETTER_TYPE
+    ).all()
    Notification.query.filter(
-        Notification.id.in_([n.id for n in notifications]),
+        Notification.created_at < timeout_start,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type != LETTER_TYPE
    ).update(
        {'status': new_status, 'updated_at': updated_at},
        synchronize_session=False
    )
+    return notifications
+
+
+def dao_timeout_notifications(timeout_period_in_seconds):
+    """
+    Timeout SMS and email notifications by the following rules:
+
+    we never sent the notification to the provider for some reason
+        created -> technical-failure
+
+    the notification was sent to the provider but there was not a delivery receipt
+        sending -> temporary-failure
+        pending -> temporary-failure
+
+    Letter notifications are not timed out
+    """
+    timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
+    updated_at = datetime.utcnow()
+    timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
+
+    # Notifications still in created status are marked with a technical-failure:
+    technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
+
+    # Notifications still in sending or pending status are marked with a temporary-failure:
+    temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
+                                              NOTIFICATION_TEMPORARY_FAILURE)

    db.session.commit()
-    return notifications
+
+    return technical_failure_notifications, temporary_failure_notifications


 def is_delivery_slow_for_providers(
@@ -469,7 +523,6 @@ def is_delivery_slow_for_providers(
        ProviderDetails
    ).outerjoin(
        Notification, and_(
-            Notification.notification_type == SMS_TYPE,
            Notification.sent_by == ProviderDetails.identifier,
            Notification.created_at >= created_at,
            Notification.sent_at.isnot(None),
@@ -493,12 +546,16 @@ def is_delivery_slow_for_providers(
        slow_notifications = sum(row.count for row in rows if row.slow)

        slow_providers[provider] = (slow_notifications / total_notifications >= threshold)
-        statsd_client.gauge(f'slow-delivery.{provider}.ratio', slow_notifications / total_notifications)
+
+        current_app.logger.info("Slow delivery notifications count for provider {}: {} out of {}. Ratio {}".format(
+            provider, slow_notifications, total_notifications, slow_notifications / total_notifications
+        ))

    return slow_providers


-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_update_notifications_by_reference(references, update_dict):
    updated_count = Notification.query.filter(
        Notification.reference.in_(references)
@@ -519,15 +576,8 @@ def dao_update_notifications_by_reference(references, update_dict):
    return updated_count, updated_history_count


-def dao_get_notifications_by_recipient_or_reference(
-    service_id,
-    search_term,
-    notification_type=None,
-    statuses=None,
-    page=1,
-    page_size=None,
-    error_out=True,
-):
+@statsd(namespace="dao")
+def dao_get_notifications_by_recipient_or_reference(service_id, search_term, notification_type=None, statuses=None):

    if notification_type == SMS_TYPE:
        normalised = try_validate_and_format_phone_number(search_term)
@@ -543,19 +593,11 @@ def dao_get_notifications_by_recipient_or_reference(
        except InvalidEmailError:
            normalised = search_term.lower()

-    elif notification_type in {LETTER_TYPE, None}:
-        # For letters, we store the address without spaces, so we need
-        # to removes spaces from the search term to match. We also do
-        # this when a notification type isn’t provided (this will
-        # happen if a user doesn’t have permission to see the dashboard)
-        # because email addresses and phone numbers will never be stored
-        # with spaces either.
-        normalised = ''.join(search_term.split()).lower()
+    elif notification_type == LETTER_TYPE:
+        raise InvalidRequest("Only email and SMS can use search by recipient", 400)

    else:
-        raise TypeError(
-            f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, {LETTER_TYPE} or None'
-        )
+        normalised = search_term.lower()

    normalised = escape_special_characters(normalised)
    search_term = escape_special_characters(search_term)
@@ -574,20 +616,19 @@ def dao_get_notifications_by_recipient_or_reference(
    if notification_type:
        filters.append(Notification.notification_type == notification_type)

-    results = db.session.query(Notification)\
-        .filter(*filters)\
-        .order_by(desc(Notification.created_at))\
-        .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
+    results = db.session.query(Notification).filter(*filters).order_by(desc(Notification.created_at)).all()
    return results


+@statsd(namespace="dao")
 def dao_get_notification_by_reference(reference):
    return Notification.query.filter(
        Notification.reference == reference
    ).one()


-def dao_get_notification_or_history_by_reference(reference):
+@statsd(namespace="dao")
+def dao_get_notification_history_by_reference(reference):
    try:
        # This try except is necessary because in test keys and research mode does not create notification history.
        # Otherwise we could just search for the NotificationHistory object
@@ -600,15 +641,45 @@ def dao_get_notification_or_history_by_reference(reference):
        ).one()


-def dao_get_notifications_processing_time_stats(start_date, end_date):
-    """
-    For a given time range, returns the number of notifications sent and the number of
-    those notifications that we processed within 10 seconds
+@statsd(namespace="dao")
+def dao_get_notifications_by_references(references):
+    return Notification.query.filter(
+        Notification.reference.in_(references)
+    ).all()

+
+@statsd(namespace="dao")
+def dao_created_scheduled_notification(scheduled_notification):
+    db.session.add(scheduled_notification)
+    db.session.commit()
+
+
+@statsd(namespace="dao")
+def dao_get_scheduled_notifications():
+    notifications = Notification.query.join(
+        ScheduledNotification
+    ).filter(
+        ScheduledNotification.scheduled_for < datetime.utcnow(),
+        ScheduledNotification.pending).all()
+
+    return notifications
+
+
+def set_scheduled_notification_to_processed(notification_id):
+    db.session.query(ScheduledNotification).filter(
+        ScheduledNotification.notification_id == notification_id
+    ).update(
+        {'pending': False}
+    )
+    db.session.commit()
+
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+    """
    SELECT
-    count(notifications),
+    count(notification_history),
    coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
    WHERE
    created_at > 'START DATE' AND
    created_at < 'END DATE' AND
@@ -638,6 +709,7 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
    ).one()


+@statsd(namespace="dao")
 def dao_get_last_notification_added_for_job_id(job_id):
    last_notification_added = Notification.query.filter(
        Notification.job_id == job_id
@@ -659,50 +731,18 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
    return notifications


-def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
+@statsd(namespace="dao")
+def dao_get_letters_to_be_printed(print_run_deadline):
    """
-    Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
-    to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
-    get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese  start sending off
-    tasks to the notify-ftp app to send them.
-
-    CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
-    there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
-    and we should avoid modifying state of returned objects.
-
-    For more reading:
-    https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
-    https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
+    Return all letters created before the print run deadline that have not yet been sent
    """
    notifications = Notification.query.filter(
        Notification.created_at < convert_bst_to_utc(print_run_deadline),
        Notification.notification_type == LETTER_TYPE,
        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.postage == postage,
-        Notification.billable_units > 0
+        Notification.key_type == KEY_TYPE_NORMAL
    ).order_by(
-        Notification.service_id,
        Notification.created_at
-    ).yield_per(query_limit)
-    return notifications
-
-
-def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
-    notifications = db.session.query(
-        func.count(Notification.id).label('letters_count'),
-        func.sum(Notification.billable_units).label('sheets_count'),
-        Notification.postage
-    ).filter(
-        Notification.created_at < convert_bst_to_utc(print_run_deadline),
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.billable_units > 0
-    ).group_by(
-        Notification.postage
-    ).order_by(
-        Notification.postage
    ).all()
    return notifications

@@ -753,58 +793,14 @@ def dao_precompiled_letters_still_pending_virus_check():
 def _duplicate_update_warning(notification, status):
    current_app.logger.info(
        (
-            'Duplicate callback received for service {service_id}. '
-            'Notification ID {id} with type {type} sent by {sent_by}. '
-            'New status was {new_status}, current status is {old_status}. '
-            'This happened {time_diff} after being first set.'
+            'Duplicate callback received. Notification id {id} received a status update to {new_status}'
+            '{time_diff} after being set to {old_status}. {type} sent by {sent_by}'
        ).format(
            id=notification.id,
            old_status=notification.status,
            new_status=status,
            time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
            type=notification.notification_type,
-            sent_by=notification.sent_by,
-            service_id=notification.service_id
+            sent_by=notification.sent_by
        )
    )
-
-
-def get_service_ids_with_notifications_before(notification_type, timestamp):
-    return {
-        row.service_id
-        for row in db.session.query(
-            Notification.service_id
-        ).filter(
-            Notification.notification_type == notification_type,
-            Notification.created_at < timestamp
-        ).distinct()
-    }
-
-
-def get_service_ids_with_notifications_on_date(notification_type, date):
-    start_date = get_london_midnight_in_utc(date)
-    end_date = get_london_midnight_in_utc(date + timedelta(days=1))
-
-    notification_table_query = db.session.query(
-        Notification.service_id.label('service_id')
-    ).filter(
-        Notification.notification_type == notification_type,
-        # using >= + < is much more efficient than date(created_at)
-        Notification.created_at >= start_date,
-        Notification.created_at < end_date,
-    )
-
-    # Looking at this table is more efficient for historical notifications,
-    # provided the task to populate it has run before they were archived.
-    ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label('service_id')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.bst_date == date,
-    )
-
-    return {
-        row.service_id for row in db.session.query(union(
-            notification_table_query, ft_status_table_query
-        ).subquery()).distinct()
-    }
--- a/app/dao/organisation_dao.py
+++ b/app/dao/organisation_dao.py
@@ -1,8 +1,14 @@
 from sqlalchemy.sql.expression import func

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.models import Domain, Organisation, Service, User
+from app.dao.dao_utils import VersionOptions, transactional, version_class
+from app.models import (
+    Organisation,
+    Domain,
+    InvitedOrganisationUser,
+    Service,
+    User
+)


 def dao_get_organisations():
@@ -55,12 +61,12 @@ def dao_get_organisation_by_service_id(service_id):
    return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()


-@autocommit
+@transactional
 def dao_create_organisation(organisation):
    db.session.add(organisation)


-@autocommit
+@transactional
 def dao_update_organisation(organisation_id, **kwargs):

    domains = kwargs.pop('domains', None)
@@ -83,9 +89,6 @@ def dao_update_organisation(organisation_id, **kwargs):
    if 'organisation_type' in kwargs:
        _update_organisation_services(organisation, 'organisation_type', only_where_none=False)

-    if 'crown' in kwargs:
-        _update_organisation_services(organisation, 'crown', only_where_none=False)
-
    if 'email_branding_id' in kwargs:
        _update_organisation_services(organisation, 'email_branding')

@@ -105,7 +108,7 @@ def _update_organisation_services(organisation, attribute, only_where_none=True)
        db.session.add(service)


-@autocommit
+@transactional
@version_class(Service)
 def dao_add_service_to_organisation(service, organisation_id):
    organisation = Organisation.query.filter_by(
@@ -119,26 +122,21 @@ def dao_add_service_to_organisation(service, organisation_id):
    db.session.add(service)


+def dao_get_invited_organisation_user(user_id):
+    return InvitedOrganisationUser.query.filter_by(id=user_id).one()
+
+
 def dao_get_users_for_organisation(organisation_id):
-    return db.session.query(
-        User
-    ).join(
-        User.organisations
-    ).filter(
-        Organisation.id == organisation_id,
+    return User.query.filter(
+        User.organisations.any(id=organisation_id),
        User.state == 'active'
    ).order_by(User.created_at).all()


-@autocommit
+@transactional
 def dao_add_user_to_organisation(organisation_id, user_id):
    organisation = dao_get_organisation_by_id(organisation_id)
    user = User.query.filter_by(id=user_id).one()
    user.organisations.append(organisation)
    db.session.add(organisation)
    return user
-
-
-@autocommit
-def dao_remove_user_from_organisation(organisation, user):
-    organisation.users.remove(user)
--- a/app/dao/permissions_dao.py
+++ b/app/dao/permissions_dao.py
@@ -1,16 +1,16 @@
 from app import db
 from app.dao import DAOClass
 from app.models import (
-    MANAGE_API_KEYS,
-    MANAGE_SETTINGS,
-    MANAGE_TEMPLATES,
+    Permission,
    MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
    SEND_EMAILS,
    SEND_LETTERS,
-    SEND_TEXTS,
-    VIEW_ACTIVITY,
-    Permission,
-)
+    MANAGE_API_KEYS,
+    VIEW_ACTIVITY)
+

 # Default permissions for a service
 default_service_permissions = [
--- a/app/dao/provider_details_dao.py
+++ b/app/dao/provider_details_dao.py
@@ -1,18 +1,12 @@
 from datetime import datetime, timedelta

-from flask import current_app
 from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import asc, desc, func
+from flask import current_app

+from app.dao.dao_utils import transactional
+from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    FactBilling,
-    ProviderDetails,
-    ProviderDetailsHistory,
-    User,
-)


 def get_provider_details_by_id(provider_details_id):
@@ -36,8 +30,6 @@ def dao_get_provider_versions(provider_id):
        id=provider_id
    ).order_by(
        desc(ProviderDetailsHistory.version)
-    ).limit(
-        100  # limit results instead of adding pagination
    ).all()


@@ -77,7 +69,7 @@ def _get_sms_providers_for_update(time_threshold):
    return q


-@autocommit
+@transactional
 def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    """
    Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
@@ -86,8 +78,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    amount_to_reduce_by = 10
    providers_list = _get_sms_providers_for_update(time_threshold)

-    if len(providers_list) < 2:
-        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
+    if not providers_list:
        return

    providers = {provider.identifier: provider for provider in providers_list}
@@ -104,7 +95,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    _adjust_provider_priority(increased_provider, increased_provider_priority)


-@autocommit
+@transactional
 def dao_adjust_provider_priority_back_to_resting_points():
    """
    Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
@@ -138,7 +129,7 @@ def get_provider_details_by_notification_type(notification_type, supports_intern
    return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()


-@autocommit
+@transactional
 def dao_update_provider_details(provider_details):
    _update_provider_details_without_commit(provider_details)

--- a/app/dao/provider_rates_dao.py
+++ b/app/dao/provider_rates_dao.py
@@ -0,0 +1,11 @@
+from app.models import ProviderRates, ProviderDetails
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
+    db.session.add(provider_rates)
--- a/app/dao/returned_letters_dao.py
+++ b/app/dao/returned_letters_dao.py
@@ -1,10 +1,10 @@
 from datetime import datetime

-from sqlalchemy import desc, func
+from sqlalchemy import func, desc
 from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import (
    Job,
    Notification,
@@ -28,7 +28,7 @@ def _get_notification_ids_for_references(references):
    return notification_ids + notification_history_ids


-@autocommit
+@transactional
 def insert_or_update_returned_letters(references):
    data = _get_notification_ids_for_references(references)
    for row in data:
--- a/app/dao/service_callback_api_dao.py
+++ b/app/dao/service_callback_api_dao.py
@@ -1,15 +1,13 @@
 from datetime import datetime

-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    COMPLAINT_CALLBACK_TYPE,
-    DELIVERY_STATUS_CALLBACK_TYPE,
-    ServiceCallbackApi,
-)
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
+from app.models import ServiceCallbackApi
+
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE


-@autocommit
+@transactional
@version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
    service_callback_api.id = create_uuid()
@@ -17,7 +15,7 @@ def save_service_callback_api(service_callback_api):
    db.session.add(service_callback_api)


-@autocommit
+@transactional
@version_class(ServiceCallbackApi)
 def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
    if url:
@@ -48,6 +46,6 @@ def get_service_complaint_callback_api_for_service(service_id):
    ).first()


-@autocommit
+@transactional
 def delete_service_callback_api(service_callback_api):
    db.session.delete(service_callback_api)
--- a/app/dao/service_contact_list_dao.py
+++ b/app/dao/service_contact_list_dao.py
@@ -1,33 +0,0 @@
-from app import db
-from app.models import ServiceContactList
-
-
-def dao_get_contact_list_by_id(service_id, contact_list_id):
-    contact_list = ServiceContactList.query.filter_by(
-        service_id=service_id,
-        id=contact_list_id,
-        archived=False,
-    ).one()
-
-    return contact_list
-
-
-def dao_get_contact_lists(service_id):
-    contact_lists = ServiceContactList.query.filter_by(
-        service_id=service_id,
-        archived=False,
-    ).order_by(
-        ServiceContactList.created_at.desc()
-    )
-    return contact_lists.all()
-
-
-def save_service_contact_list(service_contact_list):
-    db.session.add(service_contact_list)
-    db.session.commit()
-
-
-def dao_archive_contact_list(service_contact_list):
-    service_contact_list.archived = True
-    db.session.add(service_contact_list)
-    db.session.commit()
--- a/app/dao/service_data_retention_dao.py
+++ b/app/dao/service_data_retention_dao.py
@@ -1,7 +1,7 @@
 from datetime import datetime

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceDataRetention


@@ -28,7 +28,7 @@ def fetch_service_data_retention_by_notification_type(service_id, notification_t
    return data_retention_list


-@autocommit
+@transactional
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
    new_data_retention = ServiceDataRetention(service_id=service_id,
                                              notification_type=notification_type,
@@ -38,7 +38,7 @@ def insert_service_data_retention(service_id, notification_type, days_of_retenti
    return new_data_retention


-@autocommit
+@transactional
 def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
    updated_count = ServiceDataRetention.query.filter(
        ServiceDataRetention.id == service_data_retention_id,
@@ -50,7 +50,3 @@ def update_service_data_retention(service_data_retention_id, service_id, days_of
        }
    )
    return updated_count
-
-
-def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
-    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()
--- a/app/dao/service_email_reply_to_dao.py
+++ b/app/dao/service_email_reply_to_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceEmailReplyTo
@@ -28,7 +28,7 @@ def dao_get_reply_to_by_id(service_id, reply_to_id):
    return reply_to


-@autocommit
+@transactional
 def add_reply_to_email_address_for_service(service_id, email_address, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -41,7 +41,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
    return new_reply_to


-@autocommit
+@transactional
 def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -57,7 +57,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
    return reply_to_update


-@autocommit
+@transactional
 def archive_reply_to_email_address(service_id, reply_to_id):
    reply_to_archive = ServiceEmailReplyTo.query.filter_by(
        id=reply_to_id,
--- a/app/dao/service_guest_list_dao.py
+++ b/app/dao/service_guest_list_dao.py
@@ -1,17 +0,0 @@
-from app import db
-from app.models import ServiceGuestList
-
-
-def dao_fetch_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).all()
-
-
-def dao_add_and_commit_guest_list_contacts(objs):
-    db.session.add_all(objs)
-    db.session.commit()
-
-
-def dao_remove_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).delete()
--- a/app/dao/service_inbound_api_dao.py
+++ b/app/dao/service_inbound_api_dao.py
@@ -1,11 +1,11 @@
 from datetime import datetime

-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceInboundApi


-@autocommit
+@transactional
@version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
    service_inbound_api.id = create_uuid()
@@ -13,7 +13,7 @@ def save_service_inbound_api(service_inbound_api):
    db.session.add(service_inbound_api)


-@autocommit
+@transactional
@version_class(ServiceInboundApi)
 def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
    if url:
@@ -35,6 +35,6 @@ def get_service_inbound_api_for_service(service_id):
    return ServiceInboundApi.query.filter_by(service_id=service_id).first()


-@autocommit
+@transactional
 def delete_service_inbound_api(service_inbound_api):
    db.session.delete(service_inbound_api)
--- a/app/dao/service_letter_contact_dao.py
+++ b/app/dao/service_letter_contact_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceLetterContact, Template


@@ -30,7 +30,7 @@ def dao_get_letter_contact_by_id(service_id, letter_contact_id):
    return letter_contact


-@autocommit
+@transactional
 def add_letter_contact_for_service(service_id, contact_block, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -45,7 +45,7 @@ def add_letter_contact_for_service(service_id, contact_block, is_default):
    return new_letter_contact


-@autocommit
+@transactional
 def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
    old_default = _get_existing_default(service_id)
    # if we want to make this the default, ensure there are no other existing defaults
@@ -59,7 +59,7 @@ def update_letter_contact(service_id, letter_contact_id, contact_block, is_defau
    return letter_contact_update


-@autocommit
+@transactional
 def archive_letter_contact(service_id, letter_contact_id):
    letter_contact_to_archive = ServiceLetterContact.query.filter_by(
        id=letter_contact_id,
--- a/app/dao/service_permissions_dao.py
+++ b/app/dao/service_permissions_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServicePermission


@@ -8,7 +8,7 @@ def dao_fetch_service_permissions(service_id):
        ServicePermission.service_id == service_id).all()


-@autocommit
+@transactional
 def dao_add_service_permission(service_id, permission):
    service_permission = ServicePermission(service_id=service_id, permission=permission)
    db.session.add(service_permission)
--- a/app/dao/service_sms_sender_dao.py
+++ b/app/dao/service_sms_sender_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceSmsSender

@@ -32,7 +32,7 @@ def dao_get_sms_senders_by_service_id(service_id):
    ).order_by(desc(ServiceSmsSender.is_default)).all()


-@autocommit
+@transactional
 def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
    old_default = _get_existing_default(service_id=service_id)
    if is_default:
@@ -51,7 +51,7 @@ def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_n
    return new_sms_sender


-@autocommit
+@transactional
 def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -68,7 +68,7 @@ def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default,
    return sms_sender_to_update


-@autocommit
+@transactional
 def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
    service_sms_sender.sms_sender = sms_sender
    service_sms_sender.inbound_number_id = inbound_number_id
@@ -76,7 +76,7 @@ def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sende
    return service_sms_sender


-@autocommit
+@transactional
 def archive_sms_sender(service_id, sms_sender_id):
    sms_sender_to_archive = ServiceSmsSender.query.filter_by(
        id=sms_sender_id,
--- a/app/dao/service_user_dao.py
+++ b/app/dao/service_user_dao.py
@@ -1,6 +1,6 @@

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceUser, User


@@ -9,13 +9,9 @@ def dao_get_service_user(user_id, service_id):


 def dao_get_active_service_users(service_id):
-    query = db.session.query(
-        ServiceUser
-    ).join(
-        User, User.id == ServiceUser.user_id
-    ).filter(
-        User.state == 'active',
-        ServiceUser.service_id == service_id
+    query = ServiceUser.query.join(ServiceUser.user).filter(
+        ServiceUser.service_id == service_id,
+        User.state == 'active'
    )

    return query.all()
@@ -25,6 +21,6 @@ def dao_get_service_users_by_user_id(user_id):
    return ServiceUser.query.filter_by(user_id=user_id).all()


-@autocommit
+@transactional
 def dao_update_service_user(service_user):
    db.session.add(service_user)
--- a/app/dao/service_whitelist_dao.py
+++ b/app/dao/service_whitelist_dao.py
@@ -0,0 +1,17 @@
+from app import db
+from app.models import ServiceWhitelist
+
+
+def dao_fetch_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).all()
+
+
+def dao_add_and_commit_whitelisted_contacts(objs):
+    db.session.add_all(objs)
+    db.session.commit()
+
+
+def dao_remove_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).delete()
--- a/app/dao/services_dao.py
+++ b/app/dao/services_dao.py
@@ -1,14 +1,19 @@
 import uuid
 from datetime import date, datetime, timedelta

-from flask import current_app
-from sqlalchemy import Float, cast
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy.sql.expression import asc, case, and_, func
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import and_, asc, case, func
+from sqlalchemy import cast, Float
+from flask import current_app

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
 from app.dao.email_branding_dao import dao_get_email_branding_by_name
 from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
@@ -16,17 +21,6 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    CROWN_ORGANISATION_TYPES,
-    EMAIL_TYPE,
-    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_SMS_TYPE,
-    KEY_TYPE_TEST,
-    LETTER_TYPE,
-    NHS_ORGANISATION_TYPES,
-    NON_CROWN_ORGANISATION_TYPES,
-    NOTIFICATION_PERMANENT_FAILURE,
-    SMS_TYPE,
-    UPLOAD_LETTERS,
    AnnualBilling,
    ApiKey,
    FactBilling,
@@ -38,9 +32,6 @@ from app.models import (
    Organisation,
    Permission,
    Service,
-    ServiceContactList,
-    ServiceEmailReplyTo,
-    ServiceLetterContact,
    ServicePermission,
    ServiceSmsSender,
    Template,
@@ -48,13 +39,18 @@ from app.models import (
    TemplateRedacted,
    User,
    VerifyCode,
+    CROWN_ORGANISATION_TYPES,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NHS_ORGANISATION_TYPES,
+    NON_CROWN_ORGANISATION_TYPES,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    UPLOAD_LETTERS,
 )
-from app.utils import (
-    email_address_is_nhs,
-    escape_special_characters,
-    get_archived_db_column_value,
-    get_london_midnight_in_utc,
-)
+from app.utils import email_address_is_nhs, escape_special_characters, get_london_midnight_in_utc, midnight_n_days_ago

 DEFAULT_SERVICE_PERMISSIONS = [
    SMS_TYPE,
@@ -62,7 +58,6 @@ DEFAULT_SERVICE_PERMISSIONS = [
    LETTER_TYPE,
    INTERNATIONAL_SMS_TYPE,
    UPLOAD_LETTERS,
-    INTERNATIONAL_LETTERS,
 ]


@@ -236,17 +231,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False):
    return query.all()


-def dao_fetch_all_services_created_by_user(user_id):
-    query = Service.query.filter_by(
-        created_by_id=user_id
-    ).order_by(
-        asc(Service.created_at)
-    )
-
-    return query.all()
-
-
-@autocommit
+@transactional
@version_class(
    VersionOptions(ApiKey, must_write_history=False),
    VersionOptions(Service),
@@ -262,8 +247,8 @@ def dao_archive_service(service_id):
    ).filter(Service.id == service_id).one()

    service.active = False
-    service.name = get_archived_db_column_value(service.name)
-    service.email_from = get_archived_db_column_value(service.email_from)
+    service.name = '_archived_' + service.name
+    service.email_from = '_archived_' + service.email_from

    for template in service.templates:
        if not template.archived:
@@ -283,7 +268,7 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
    ).one()


-@autocommit
+@transactional
@version_class(Service)
 def dao_create_service(
    service,
@@ -291,6 +276,9 @@ def dao_create_service(
    service_id=None,
    service_permissions=None,
 ):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.

    if not user:
        raise ValueError("Can't create a service without a user")
@@ -317,17 +305,15 @@ def dao_create_service(
    if organisation:
        service.organisation_id = organisation.id
        service.organisation_type = organisation.organisation_type
-
        if organisation.email_branding:
            service.email_branding = organisation.email_branding

-        if organisation.letter_branding:
+        if organisation.letter_branding and not service.letter_branding:
            service.letter_branding = organisation.letter_branding

    elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
        service.email_branding = dao_get_email_branding_by_name('NHS')
        service.letter_branding = dao_get_letter_branding_by_name('NHS')
-
    if organisation:
        service.crown = organisation.crown
    elif service.organisation_type in CROWN_ORGANISATION_TYPES:
@@ -339,7 +325,7 @@ def dao_create_service(
    db.session.add(service)


-@autocommit
+@transactional
@version_class(Service)
 def dao_update_service(service):
    db.session.add(service)
@@ -391,9 +377,6 @@ def delete_service_and_all_associated_db_objects(service):
    _delete_commit(TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq)))

    _delete_commit(ServiceSmsSender.query.filter_by(service=service))
-    _delete_commit(ServiceEmailReplyTo.query.filter_by(service=service))
-    _delete_commit(ServiceLetterContact.query.filter_by(service=service))
-    _delete_commit(ServiceContactList.query.filter_by(service=service))
    _delete_commit(InvitedUser.query.filter_by(service=service))
    _delete_commit(Permission.query.filter_by(service=service))
    _delete_commit(NotificationHistory.query.filter_by(service=service))
@@ -410,35 +393,60 @@ def delete_service_and_all_associated_db_objects(service):
    list(map(db.session.delete, verify_codes))
    db.session.commit()
    users = [x for x in service.users]
-    for user in users:
-        user.organisations = []
-        service.users.remove(user)
+    map(service.users.remove, users)
+    [service.users.remove(x) for x in users]
    _delete_commit(Service.get_history_model().query.filter_by(id=service.id))
    db.session.delete(service)
    db.session.commit()
-    for user in users:
-        db.session.delete(user)
+    list(map(db.session.delete, users))
    db.session.commit()


-def dao_fetch_todays_stats_for_service(service_id):
-    today = date.today()
-    start_date = get_london_midnight_in_utc(today)
+@statsd(namespace="dao")
+def dao_fetch_stats_for_service(service_id, limit_days):
+    # We always want between seven and eight days
+    start_date = midnight_n_days_ago(limit_days)
+    return _stats_for_service_query(service_id).filter(
+        Notification.created_at >= start_date
+    ).all()

+
+@statsd(namespace="dao")
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def fetch_todays_total_message_count(service_id):
+    result = db.session.query(
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        func.date(Notification.created_at) == date.today()
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).first()
+    return 0 if result is None else result.count
+
+
+def _stats_for_service_query(service_id):
    return db.session.query(
        Notification.notification_type,
        Notification.status,
        func.count(Notification.id).label('count')
    ).filter(
        Notification.service_id == service_id,
-        Notification.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= start_date
+        Notification.key_type != KEY_TYPE_TEST
    ).group_by(
        Notification.notification_type,
        Notification.status,
-    ).all()
+    )


+@statsd(namespace='dao')
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
    today = date.today()
    start_date = get_london_midnight_in_utc(today)
@@ -484,7 +492,7 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act
    return query.all()


-@autocommit
+@transactional
@version_class(
    VersionOptions(ApiKey, must_write_history=False),
    VersionOptions(Service),
@@ -503,7 +511,7 @@ def dao_suspend_service(service_id):
    service.active = False


-@autocommit
+@transactional
@version_class(Service)
 def dao_resume_service(service_id):
    service = Service.query.get(service_id)
@@ -531,8 +539,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
        Notification.notification_type == SMS_TYPE,
        func.substr(Notification.normalised_to, 3, 7) == '7700900',
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
    ).having(
@@ -540,7 +548,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
    ).all()


-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
    subquery = db.session.query(
        func.count(Notification.id).label('total_count'),
        Notification.service_id.label('service_id')
@@ -551,8 +559,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
        Notification.key_type != KEY_TYPE_TEST,
        Notification.notification_type == SMS_TYPE,
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
    ).having(
@@ -577,8 +585,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
        Notification.notification_type == SMS_TYPE,
        Notification.status == NOTIFICATION_PERMANENT_FAILURE,
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
        subquery.c.total_count
@@ -587,23 +595,3 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
    )

    return query.all()
-
-
-def get_live_services_with_organisation():
-    query = db.session.query(
-        Service.id.label("service_id"),
-        Service.name.label("service_name"),
-        Organisation.id.label("organisation_id"),
-        Organisation.name.label("organisation_name")
-    ).outerjoin(
-        Service.organisation
-    ).filter(
-        Service.count_as_live.is_(True),
-        Service.active.is_(True),
-        Service.restricted.is_(False)
-    ).order_by(
-        Organisation.name,
-        Service.name
-    )
-
-    return query.all()
--- a/app/dao/template_folder_dao.py
+++ b/app/dao/template_folder_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import TemplateFolder


@@ -14,16 +14,16 @@ def dao_get_valid_template_folders_by_id(folder_ids):
    return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all()


-@autocommit
+@transactional
 def dao_create_template_folder(template_folder):
    db.session.add(template_folder)


-@autocommit
+@transactional
 def dao_update_template_folder(template_folder):
    db.session.add(template_folder)


-@autocommit
+@transactional
 def dao_delete_template_folder(template_folder):
    db.session.delete(template_folder)
--- a/app/dao/templates_dao.py
+++ b/app/dao/templates_dao.py
@@ -1,22 +1,26 @@
-import uuid
 from datetime import datetime
+import uuid

 from flask import current_app
 from sqlalchemy import asc, desc

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.users_dao import get_user_by_id
 from app.models import (
    LETTER_TYPE,
    SECOND_CLASS,
    Template,
    TemplateHistory,
-    TemplateRedacted,
+    TemplateRedacted
 )
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.users_dao import get_user_by_id


-@autocommit
+@transactional
@version_class(
    VersionOptions(Template, history_class=TemplateHistory)
 )
@@ -38,15 +42,18 @@ def dao_create_template(template):
    db.session.add(template)


-@autocommit
+@transactional
@version_class(
    VersionOptions(Template, history_class=TemplateHistory)
 )
 def dao_update_template(template):
+    if template.archived:
+        template.folder = None
+
    db.session.add(template)


-@autocommit
+@transactional
 def dao_update_template_reply_to(template_id, reply_to):
    Template.query.filter_by(id=template_id).update(
        {"service_letter_contact_id": reply_to,
@@ -71,14 +78,13 @@ def dao_update_template_reply_to(template_id, reply_to):
                                  "version": template.version,
                                  "archived": template.archived,
                                  "process_type": template.process_type,
-                                  "service_letter_contact_id": template.service_letter_contact_id,
-                                  "broadcast_data": template.broadcast_data,
+                                  "service_letter_contact_id": template.service_letter_contact_id
                              })
    db.session.add(history)
    return template


-@autocommit
+@transactional
 def dao_redact_template(template, user_id):
    template.template_redacted.redact_personalisation = True
    template.template_redacted.updated_at = datetime.utcnow()
--- a/app/dao/uploads_dao.py
+++ b/app/dao/uploads_dao.py
@@ -1,44 +1,15 @@
 from datetime import datetime
-
 from flask import current_app
-from sqlalchemy import String, and_, desc, func, literal, text
+from sqlalchemy import and_, desc, func, literal, String

 from app import db
 from app.models import (
-    JOB_STATUS_CANCELLED,
-    JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    Job,
-    Notification,
-    ServiceDataRetention,
-    Template,
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED, ServiceDataRetention
 )
 from app.utils import midnight_n_days_ago


-def _get_printing_day(created_at):
-    return func.date_trunc(
-        'day',
-        func.timezone('Europe/London', func.timezone('UTC', created_at)) + text(
-            # We add 6 hours 30 minutes to the local created_at time so that
-            # any letters created after 5:30pm get shifted into the next day
-            "interval '6 hours 30 minutes'"
-        )
-    )
-
-
-def _get_printing_datetime(created_at):
-    return _get_printing_day(created_at) + text(
-        # Letters are printed from 5:30pm each day
-        "interval '17 hours 30 minutes'"
-    )
-
-
-def _naive_gmt_to_utc(column):
-    return func.timezone('UTC', func.timezone('Europe/London', column))
-
-
 def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50):
    # Hardcoded filter to exclude cancelled or scheduled jobs
    # for the moment, but we may want to change this method take 'statuses' as a argument in the future
@@ -50,8 +21,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
        func.coalesce(
            Job.processing_started, Job.created_at
-        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
-        Job.contact_list_id.is_(None),
+        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
    ]
    if limit_days is not None:
        jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
@@ -84,15 +54,26 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
        Notification.notification_type == LETTER_TYPE,
        Notification.api_key_id == None,  # noqa
        Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
+        Template.hidden == True,
        Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
+
    ]
    if limit_days is not None:
        letters_query_filter.append(Notification.created_at >= midnight_n_days_ago(limit_days))

-    letters_subquery = db.session.query(
-        func.count().label('notification_count'),
-        _naive_gmt_to_utc(_get_printing_datetime(Notification.created_at)).label('printing_at'),
+    letters_query = db.session.query(
+        Notification.id,
+        Notification.client_reference.label('original_file_name'),
+        literal('1').label('notification_count'),
+        literal(None).label('template_type'),
+        func.coalesce(ServiceDataRetention.days_of_retention, 7).label('days_of_retention'),
+        Notification.created_at.label("created_at"),
+        literal(None).label('scheduled_for'),
+        # letters don't have a processing_started date but we want created_at to be used for sorting
+        Notification.created_at.label('processing_started'),
+        Notification.status,
+        literal('letter').label('upload_type'),
+        Notification.to.label('recipient'),
    ).join(
        Template, Notification.template_id == Template.id
    ).outerjoin(
@@ -102,25 +83,6 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
        )
    ).filter(
        *letters_query_filter
-    ).group_by(
-        'printing_at'
-    ).subquery()
-
-    letters_query = db.session.query(
-        literal(None).label('id'),
-        literal('Uploaded letters').label('original_file_name'),
-        letters_subquery.c.notification_count.label('notification_count'),
-        literal('letter').label('template_type'),
-        literal(None).label('days_of_retention'),
-        letters_subquery.c.printing_at.label('created_at'),
-        literal(None).label('scheduled_for'),
-        letters_subquery.c.printing_at.label('processing_started'),
-        literal(None).label('status'),
-        literal('letter_day').label('upload_type'),
-        literal(None).label('recipient'),
-    ).group_by(
-        letters_subquery.c.notification_count,
-        letters_subquery.c.printing_at,
    )

    return jobs_query.union_all(
@@ -128,23 +90,3 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
    ).order_by(
        desc("processing_started"), desc("created_at")
    ).paginate(page=page, per_page=page_size)
-
-
-def dao_get_uploaded_letters_by_print_date(service_id, letter_print_date, page=1, page_size=50):
-    return db.session.query(
-        Notification,
-    ).join(
-        Template, Notification.template_id == Template.id
-    ).filter(
-        Notification.service_id == service_id,
-        Notification.notification_type == LETTER_TYPE,
-        Notification.api_key_id.is_(None),
-        Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden.is_(True),
-        _get_printing_day(Notification.created_at) == letter_print_date.date(),
-    ).order_by(
-        desc(Notification.created_at)
-    ).paginate(
-        page=page,
-        per_page=page_size,
-    )
--- a/app/dao/users_dao.py
+++ b/app/dao/users_dao.py
@@ -1,17 +1,17 @@
+from random import (SystemRandom)
+from datetime import (datetime, timedelta)
 import uuid
-from datetime import datetime, timedelta
-from random import SystemRandom

 from sqlalchemy import func
 from sqlalchemy.orm import joinedload

 from app import db
-from app.dao.dao_utils import autocommit
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_users_by_user_id
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
-from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
-from app.utils import escape_special_characters, get_archived_db_column_value
+from app.models import (EMAIL_AUTH_TYPE, User, VerifyCode)
+from app.utils import escape_special_characters


 def _remove_values_for_keys_if_present(dict, keys):
@@ -20,23 +20,15 @@ def _remove_values_for_keys_if_present(dict, keys):


 def create_secret_code():
-    return ''.join(get_non_repeating_random_digits(5))
+    return ''.join(map(str, [SystemRandom().randrange(10) for i in range(5)]))


-def get_non_repeating_random_digits(length):
-    output = [None] * length
-    for index in range(length):
-        while output[index] in {None, output[index - 1]}:
-            output[index] = str(SystemRandom().randrange(10))
-    return output
-
-
-def save_user_attribute(usr, update_dict=None):
-    db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
+def save_user_attribute(usr, update_dict={}):
+    db.session.query(User).filter_by(id=usr.id).update(update_dict)
    db.session.commit()


-def save_model_user(user, update_dict=None, password=None, validated_email_access=False):
+def save_model_user(user, update_dict={}, password=None, validated_email_access=False):
    if password:
        user.password = password
        user.password_changed_at = datetime.utcnow()
@@ -44,7 +36,7 @@ def save_model_user(user, update_dict=None, password=None, validated_email_acces
        user.email_access_validated_at = datetime.utcnow()
    if update_dict:
        _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
-        db.session.query(User).filter_by(id=user.id).update(update_dict or {})
+        db.session.query(User).filter_by(id=user.id).update(update_dict)
    else:
        db.session.add(user)
    db.session.commit()
@@ -131,10 +123,12 @@ def reset_failed_login_count(user):
        db.session.commit()


-def update_user_password(user, password):
+def update_user_password(user, password, validated_email_access=False):
    # reset failed login count - they've just reset their password so should be fine
    user.password = password
    user.password_changed_at = datetime.utcnow()
+    if validated_email_access:
+        user.email_access_validated_at = datetime.utcnow()
    db.session.add(user)
    db.session.commit()

@@ -152,7 +146,7 @@ def get_user_and_accounts(user_id):
    ).one()


-@autocommit
+@transactional
 def dao_archive_user(user):
    if not user_can_be_archived(user):
        msg = "User can’t be removed from a service - check all services have another team member with manage_settings"
@@ -167,7 +161,7 @@ def dao_archive_user(user):
    user.organisations = []

    user.auth_type = EMAIL_AUTH_TYPE
-    user.email_address = get_archived_db_column_value(user.email_address)
+    user.email_address = get_archived_email_address(user.email_address)
    user.mobile_number = None
    user.password = str(uuid.uuid4())
    # Changing the current_session_id signs the user out
@@ -191,3 +185,8 @@ def user_can_be_archived(user):
            return False

    return True
+
+
+def get_archived_email_address(email_address):
+    date = datetime.utcnow().strftime("%Y-%m-%d")
+    return '_archived_{}_{}'.format(date, email_address)
--- a/app/dao/webauthn_credential_dao.py
+++ b/app/dao/webauthn_credential_dao.py
@@ -1,40 +0,0 @@
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import WebauthnCredential
-
-
-def dao_get_webauthn_credential_by_user_and_id(user_id, webauthn_credential_id):
-    return WebauthnCredential.query.filter(
-        WebauthnCredential.user_id == user_id,
-        WebauthnCredential.id == webauthn_credential_id
-    ).one()
-
-
-@autocommit
-def dao_create_webauthn_credential(
-    *,
-    user_id,
-    name,
-    credential_data,
-    registration_response,
-):
-    webauthn_credential = WebauthnCredential(
-        user_id=user_id,
-        name=name,
-        credential_data=credential_data,
-        registration_response=registration_response
-    )
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_update_webauthn_credential_name(webauthn_credential, new_name):
-    webauthn_credential.name = new_name
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_delete_webauthn_credential(webauthn_credential):
-    db.session.delete(webauthn_credential)
--- a/app/delivery/send_to_providers.py
+++ b/app/delivery/send_to_providers.py
@@ -1,43 +1,39 @@
 import random
-from datetime import datetime, timedelta
 from urllib import parse
+from datetime import datetime, timedelta

-from cachetools import TTLCache, cached
 from flask import current_app
-from notifications_utils.template import (
-    HTMLEmailTemplate,
-    PlainTextEmailTemplate,
-    SMSMessageTemplate,
+from notifications_utils.recipients import (
+    validate_and_format_phone_number,
+    validate_and_format_email_address
 )
+from notifications_utils.template import HTMLEmailTemplate, PlainTextEmailTemplate, SMSMessageTemplate

-from app import create_uuid, db, notification_provider_clients, statsd_client
-from app.celery.research_mode_tasks import (
-    send_email_response,
-    send_sms_response,
+from app import clients, statsd_client, create_uuid
+from app.dao.notifications_dao import (
+    dao_update_notification
 )
-from app.dao.email_branding_dao import dao_get_email_branding_by_id
-from app.dao.notifications_dao import dao_update_notification
 from app.dao.provider_details_dao import (
-    dao_reduce_sms_provider_priority,
    get_provider_details_by_notification_type,
+    dao_reduce_sms_provider_priority
 )
+from app.celery.research_mode_tasks import send_sms_response, send_email_response
+from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
+    SMS_TYPE,
+    KEY_TYPE_TEST,
    BRANDING_BOTH,
    BRANDING_ORG_BANNER,
    EMAIL_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
    NOTIFICATION_TECHNICAL_FAILURE,
-    SMS_TYPE,
+    NOTIFICATION_SENT,
+    NOTIFICATION_SENDING
 )
-from app.serialised_models import SerialisedService, SerialisedTemplate


 def send_sms_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
+    service = notification.service

    if not service.active:
        technical_failure(notification=notification)
@@ -45,13 +41,8 @@ def send_sms_to_provider(notification):

    if notification.status == 'created':
        provider = provider_to_use(SMS_TYPE, notification.international)
-        if not provider:
-            technical_failure(notification=notification)
-            return

-        template_model = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        )
+        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)

        template = SMSMessageTemplate(
            template_model.__dict__,
@@ -59,63 +50,41 @@ def send_sms_to_provider(notification):
            prefix=service.name,
            show_prefix=service.prefix_sms,
        )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
            update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.get_name(), str(notification.id), notification.to)

        else:
            try:
-                # End DB session here so that we don't have a connection stuck open waiting on the call
-                # to one of the SMS providers
-                # We don't want to tie our DB connections being open to the performance of our SMS
-                # providers as a slow down of our providers can cause us to run out of DB connections
-                # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
-                # closing the session (as otherwise it would be reopened immediately)
-                send_sms_kwargs = {
-                    'to': notification.normalised_to,
-                    'content': str(template),
-                    'reference': str(notification.id),
-                    'sender': notification.reply_to_text,
-                    'international': notification.international,
-                }
-                db.session.close()  # no commit needed as no changes to objects have been made above
-                provider.send_sms(**send_sms_kwargs)
+                provider.send_sms(
+                    to=validate_and_format_phone_number(notification.to, international=notification.international),
+                    content=str(template),
+                    reference=str(notification.id),
+                    sender=notification.reply_to_text
+                )
            except Exception as e:
                notification.billable_units = template.fragment_count
                dao_update_notification(notification)
-                dao_reduce_sms_provider_priority(provider.name, time_threshold=timedelta(minutes=1))
+                dao_reduce_sms_provider_priority(provider.get_name(), time_threshold=timedelta(minutes=1))
                raise e
            else:
                notification.billable_units = template.fragment_count
                update_notification_to_sending(notification, provider)

-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
-        statsd_client.timing("sms.total-time", delta_seconds)
-
-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("sms.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("sms.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("sms.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("sms.live-key.not-high-volume.total-time", delta_seconds)
+        delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
+        statsd_client.timing("sms.total-time", delta_milliseconds)


 def send_email_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
-
+    service = notification.service
    if not service.active:
        technical_failure(notification=notification)
        return
    if notification.status == 'created':
        provider = provider_to_use(EMAIL_TYPE)

-        template_dict = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        ).__dict__
+        template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__

        html_email = HTMLEmailTemplate(
            template_dict,
@@ -127,8 +96,7 @@ def send_email_to_provider(notification):
            template_dict,
            values=notification.personalisation
        )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
            notification.reference = str(create_uuid())
            update_notification_to_sending(notification, provider)
@@ -137,45 +105,33 @@ def send_email_to_provider(notification):
            from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
                                                 current_app.config['NOTIFY_EMAIL_DOMAIN'])

+            email_reply_to = notification.reply_to_text
+
            reference = provider.send_email(
                from_address,
-                notification.normalised_to,
+                validate_and_format_email_address(notification.to),
                plain_text_email.subject,
                body=str(plain_text_email),
                html_body=str(html_email),
-                reply_to_address=notification.reply_to_text
+                reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
            )
            notification.reference = reference
            update_notification_to_sending(notification, provider)
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()

-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("email.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("email.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
+        delta_milliseconds = (datetime.utcnow() - notification.created_at).total_seconds() * 1000
+        statsd_client.timing("email.total-time", delta_milliseconds)


 def update_notification_to_sending(notification, provider):
    notification.sent_at = datetime.utcnow()
-    notification.sent_by = provider.name
-    if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
-        notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
+    notification.sent_by = provider.get_name()
+    notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
    dao_update_notification(notification)


-provider_cache = TTLCache(maxsize=8, ttl=10)
-
-
-@cached(cache=provider_cache)
-def provider_to_use(notification_type, international=True):
-    international = False # TODO: remove or resolve the functionality of this flag
-    # TODO rip firetext and mmg out of early migrations and clean up the expression below
+def provider_to_use(notification_type, international=False):
    active_providers = [
-        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg']
+        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active
    ]

    if not active_providers:
@@ -184,13 +140,9 @@ def provider_to_use(notification_type, international=True):
        )
        raise Exception("No active {} providers".format(notification_type))

-    if len(active_providers) == 1:
-        chosen_provider = active_providers[0]
-    else:
-        weights = [p.priority for p in active_providers]
-        chosen_provider = random.choices(active_providers, weights=weights)[0]
+    chosen_provider = random.choices(active_providers, weights=[p.priority for p in active_providers])[0]

-    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
+    return clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)


 def get_logo_url(base_url, logo_file):
@@ -215,28 +167,25 @@ def get_logo_url(base_url, logo_file):


 def get_html_email_options(service):
+
    if service.email_branding is None:
        return {
            'govuk_banner': True,
            'brand_banner': False,
        }
-    if isinstance(service, SerialisedService):
-        branding = dao_get_email_branding_by_id(service.email_branding)
-    else:
-        branding = service.email_branding

    logo_url = get_logo_url(
        current_app.config['ADMIN_BASE_URL'],
-        branding.logo
-    ) if branding.logo else None
+        service.email_branding.logo
+    ) if service.email_branding.logo else None

    return {
-        'govuk_banner': branding.brand_type == BRANDING_BOTH,
-        'brand_banner': branding.brand_type == BRANDING_ORG_BANNER,
-        'brand_colour': branding.colour,
+        'govuk_banner': service.email_branding.brand_type == BRANDING_BOTH,
+        'brand_banner': service.email_branding.brand_type == BRANDING_ORG_BANNER,
+        'brand_colour': service.email_branding.colour,
        'brand_logo': logo_url,
-        'brand_text': branding.text,
-        'brand_name': branding.name,
+        'brand_text': service.email_branding.text,
+        'brand_name': service.email_branding.name,
    }


--- a/app/email_branding/email_branding_schema.py
+++ b/app/email_branding/email_branding_schema.py
@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES

 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST schema for getting email_branding",
    "type": "object",
    "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }

 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST schema for getting email_branding",
    "type": "object",
    "properties": {
--- a/app/email_branding/rest.py
+++ b/app/email_branding/rest.py
@@ -2,16 +2,16 @@ from flask import Blueprint, jsonify, request

 from app.dao.email_branding_dao import (
    dao_create_email_branding,
-    dao_get_email_branding_by_id,
    dao_get_email_branding_options,
-    dao_update_email_branding,
-)
-from app.email_branding.email_branding_schema import (
-    post_create_email_branding_schema,
-    post_update_email_branding_schema,
+    dao_get_email_branding_by_id,
+    dao_update_email_branding
 )
 from app.errors import register_errors
 from app.models import EmailBranding
+from app.email_branding.email_branding_schema import (
+    post_create_email_branding_schema,
+    post_update_email_branding_schema
+)
 from app.schema_validation import validate

 email_branding_blueprint = Blueprint('email_branding', __name__)
--- a/app/errors.py
+++ b/app/errors.py
@@ -1,10 +1,12 @@
-from flask import current_app, json, jsonify
-from jsonschema import ValidationError as JsonSchemaValidationError
-from marshmallow import ValidationError
+from flask import (
+    jsonify,
+    current_app,
+    json)
 from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
-
+from marshmallow import ValidationError
+from jsonschema import ValidationError as JsonSchemaValidationError
 from app.authentication.auth import AuthError
 from app.exceptions import ArchiveValidationError

--- a/app/events/rest.py
+++ b/app/events/rest.py
@@ -1,8 +1,13 @@
-from flask import Blueprint, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)

-from app.dao.events_dao import dao_create_event
 from app.errors import register_errors
+
 from app.schemas import event_schema
+from app.dao.events_dao import dao_create_event

 events = Blueprint('events', __name__, url_prefix='/events')
 register_errors(events)
@@ -11,6 +16,6 @@ register_errors(events)
@events.route('', methods=['POST'])
 def create_event():
    data = request.get_json()
-    event = event_schema.load(data)
+    event = event_schema.load(data).data
    dao_create_event(event)
-    return jsonify(data=event_schema.dump(event)), 201
+    return jsonify(data=event_schema.dump(event).data), 201
--- a/app/govuk_alerts/rest.py
+++ b/app/govuk_alerts/rest.py
@@ -1,31 +0,0 @@
-from flask import Blueprint, jsonify
-
-from app.dao.broadcast_message_dao import dao_get_all_broadcast_messages
-from app.errors import register_errors
-from app.utils import get_dt_string_or_none
-
-govuk_alerts_blueprint = Blueprint(
-    "govuk-alerts",
-    __name__,
-    url_prefix='/govuk-alerts',
-)
-
-register_errors(govuk_alerts_blueprint)
-
-
-@govuk_alerts_blueprint.route('')
-def get_broadcasts():
-    broadcasts = dao_get_all_broadcast_messages()
-    broadcasts_dict = {"alerts": [{
-        "id": broadcast.id,
-        "reference": broadcast.reference,
-        "channel": broadcast.channel,
-        "content": broadcast.content,
-        "areas": broadcast.areas,
-        "status": broadcast.status,
-        "starts_at": get_dt_string_or_none(broadcast.starts_at),
-        "finishes_at": get_dt_string_or_none(broadcast.finishes_at),
-        "approved_at": get_dt_string_or_none(broadcast.approved_at),
-        "cancelled_at": get_dt_string_or_none(broadcast.cancelled_at),
-    } for broadcast in broadcasts]}
-    return jsonify(broadcasts_dict), 200
--- a/app/hashing.py
+++ b/app/hashing.py
@@ -1,4 +1,4 @@
-from flask_bcrypt import check_password_hash, generate_password_hash
+from flask_bcrypt import generate_password_hash, check_password_hash


 def hashpw(password):
--- a/app/history_meta.py
+++ b/app/history_meta.py
@@ -15,11 +15,11 @@ session events.

 """
 import datetime
-
-from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, util
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import attributes, mapper, object_mapper
-from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty
+from sqlalchemy.orm import mapper, attributes, object_mapper
+from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty
+from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
+from sqlalchemy import util


 def col_references_table(col, table):
--- a/app/inbound_number/rest.py
+++ b/app/inbound_number/rest.py
@@ -1,10 +1,10 @@
 from flask import Blueprint, jsonify

 from app.dao.inbound_numbers_dao import (
-    dao_get_available_inbound_numbers,
-    dao_get_inbound_number_for_service,
    dao_get_inbound_numbers,
-    dao_set_inbound_number_active_flag,
+    dao_get_inbound_number_for_service,
+    dao_get_available_inbound_numbers,
+    dao_set_inbound_number_active_flag
 )
 from app.errors import register_errors

--- a/Show More
+++ b/Show More