Remove space between queues in paas_app_wrapper.sh

2026-08-21 14:59:26 -04:00 · 2020-10-27 16:36:37 +00:00
453 changed files with 12790 additions and 23469 deletions
--- a/.flake8
+++ b/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
+# W503: line break before binary operator
+exclude = venv*,__pycache__,node_modules,cache,migrations,build
+ignore = W503
+max-complexity = 14
+max-line-length = 120
--- a/.github/actions/setup-project/action.yml
+++ b/.github/actions/setup-project/action.yml
@@ -1,15 +0,0 @@
-name: Set up project
-description: Setup python & install dependencies
-runs:
-  using: composite
-  steps:
-    - name: Install container dependencies
-      shell: bash
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -1,125 +0,0 @@
-name: Run checks
-
-on: [push]
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  ANTIVIRUS_ENABLED: 0
-  NOTIFY_ENVIRONMENT: test
-  STATSD_HOST: localhost
-  SES_STUB_URL: None
-  NOTIFY_APP_NAME: api
-  NOTIFY_EMAIL_DOMAIN: dispostable.com
-  NOTIFY_LOG_PATH: /workspace/logs/app.log
-  ADMIN_CLIENT_ID: notify-admin
-  ADMIN_CLIENT_SECRET: dev-notify-secret-key
-  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
-  FLASK_APP: application.py
-  FLASK_ENV: development
-  WERKZEUG_DEBUG_PIN: off
-  ADMIN_BASE_URL: http://localhost:6012
-  API_HOST_NAME: http://localhost:6011
-  REDIS_URL: redis://localhost:6380
-  REDIS_ENABLED: False
-  AWS_REGION: us-west-2
-  AWS_PINPOINT_REGION: us-west-2
-  AWS_US_TOLL_FREE_NUMBER: +18446120782
-  AWS_ACCESS_KEY_ID: not-a-real-key-id
-  AWS_SECRET_ACCESS_KEY: not-a-real-secret
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: ./.github/actions/setup-project
-    - name: Install application dependencies
-      run: make bootstrap
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-    # - name: Run style checks
-    #   run: flake8 .
-    - name: Check imports alphabetized
-      run: isort --check-only ./app ./tests
-    - name: Run tests
-      run: pytest -n4 --maxfail=10
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-
-  pip-audit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - uses: trailofbits/gh-action-pip-audit@v1.0.0
-        with:
-          inputs: requirements.txt requirements_for_test.txt
-          ignore-vulns: PYSEC-2022-237
-
-  static-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install bandit
-        run: pip install bandit
-      - name: Run scan
-        run: bandit -r app/ --confidence-level medium
-
-  dynamic-scan:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install application dependencies
-        run: make bootstrap
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run server
-        run: make run-flask &
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP Baseline Scan
-        uses: zaproxy/action-api-scan@v0.1.1
-        with:
-          docker_name: 'owasp/zap2docker-stable'
-          target: 'http://localhost:6011/_status'
-          fail_action: true
-          allow_issue_writing: false
-          rules_file_name: 'zap.conf'
-          cmd_options: '-I'
--- a/.github/workflows/daily_checks.yml
+++ b/.github/workflows/daily_checks.yml
@@ -1,92 +0,0 @@
-name: Run daily scans
-
-on:
-  schedule:
-    # cron format: 'minute hour dayofmonth month dayofweek'
-    # this will run at noon UTC every day (7am EST / 8am EDT)
-    - cron: '0 12 * * *'
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  ANTIVIRUS_ENABLED: 0
-  NOTIFY_ENVIRONMENT: test
-  STATSD_HOST: localhost
-  SES_STUB_URL: None
-  NOTIFY_APP_NAME: api
-  NOTIFY_EMAIL_DOMAIN: dispostable.com
-  NOTIFY_LOG_PATH: /workspace/logs/app.log
-  ADMIN_CLIENT_ID: notify-admin
-  ADMIN_CLIENT_SECRET: dev-notify-secret-key
-  GOVUK_ALERTS_CLIENT_ID: govuk-alerts
-  FLASK_APP: application.py
-  FLASK_ENV: development
-  WERKZEUG_DEBUG_PIN: off
-  ADMIN_BASE_URL: http://localhost:6012
-  API_HOST_NAME: http://localhost:6011
-  REDIS_URL: redis://localhost:6380
-  REDIS_ENABLED: False
-  AWS_REGION: us-west-2
-  AWS_PINPOINT_REGION: us-west-2
-  AWS_US_TOLL_FREE_NUMBER: +18446120782
-
-jobs:
-  pip-audit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - uses: trailofbits/gh-action-pip-audit@v1.0.0
-        with:
-          inputs: requirements.txt requirements_for_test.txt
-          ignore-vulns: PYSEC-2022-237
-
-  static-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install bandit
-        run: pip install bandit
-      - name: Run scan
-        run: bandit -r app/ --confidence-level medium
-
-  dynamic-scan:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install application dependencies
-        run: make bootstrap
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run server
-        run: make run-flask &
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP Baseline Scan
-        uses: zaproxy/action-api-scan@v0.1.1
-        with:
-          docker_name: 'owasp/zap2docker-weekly'
-          target: 'http://localhost:6011/_status'
-          fail_action: true
-          allow_issue_writing: false
-          rules_file_name: 'zap.conf'
-          cmd_options: '-I'
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -1,84 +0,0 @@
-name: Deploy to prototype environment
-
-on:
-  workflow_run:
-    workflows: [ Run checks ]
-    types:
-      - completed
-    branches: [ main ] # Redundant, workflow_run events are only triggered on default branch (`main`)
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-
-    steps:
-    - name: Install container dependencies
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-
-    - name: Check for changes to Terraform
-      id: changed-terraform-files
-      uses: tj-actions/changed-files@v1.1.2
-      with:
-        files: terraform/staging
-    - name: Terraform init
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/staging
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-      run: terraform init
-    - name: Terraform apply
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/staging
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-        TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-      run: terraform apply -auto-approve -input=false
-
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-
-    - name: Install application dependencies
-      run: make bootstrap
-
-    - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
-      env:
-        DANGEROUS_SALT: ${{ secrets.PROD_DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.PROD_SECRET_KEY }}
-        ADMIN_CLIENT_SECRET: ${{ secrets.PROD_ADMIN_CLIENT_SECRET }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      with:
-        cf_username: ${{ secrets.cloudgov_username }}
-        cf_password: ${{ secrets.cloudgov_password }}
-        cf_org: gsa-10x-prototyping
-        cf_space: 10x-notifications
-        push_arguments: >-
-          --var env=staging
-          --var DANGEROUS_SALT="$DANGEROUS_SALT"
-          --var SECRET_KEY="$SECRET_KEY"
-          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-          --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
-          --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
-
-  bail:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - run: echo 'Checks failed, not deploying'
--- a/.github/workflows/terraform-production.yml
+++ b/.github/workflows/terraform-production.yml
@@ -1,79 +0,0 @@
-name: Run Terraform plan in production
-
-on:
-  pull_request:
-    branches: [ production ]
-    paths: [ 'terraform/**' ]
-
-defaults:
-  run:
-    working-directory: terraform/production
-
-jobs:
-  terraform:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    environment: production
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Terraform format
-        id: format
-        run: terraform fmt -check
-
-      - name: Terraform init
-        id: init
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-
-      - name: Terraform validate
-        id: validation
-        run: terraform validate -no-color
-
-      - name: Terraform plan
-        id: plan
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
-        run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
-
-      - name: Read Terraform plan output file
-        id: terraform_output
-        uses: juliangruber/read-file-action@v1
-        if: ${{ always() }}
-        with:
-          path: ./terraform/production/plan_output.txt
-
-      # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
-      - name: Update PR
-        uses: actions/github-script@v4
-        # we would like to update the PR even when a prior step failed
-        if: ${{ always() }}
-        with:
-          script: |
-            const output = `Terraform Format and Style: ${{ steps.format.outcome }}
-            Terraform Initialization: ${{ steps.init.outcome }}
-            Terraform Validation: ${{ steps.validation.outcome }}
-            Terraform Plan: ${{ steps.plan.outcome }}
-
-            <details><summary>Show Plan</summary>
-
-            \`\`\`\n
-            ${{ steps.terraform_output.outputs.content }}
-            \`\`\`
-
-            </details>
-
-            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
-
-            github.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: output
-            })
--- a/.github/workflows/terraform-staging.yml
+++ b/.github/workflows/terraform-staging.yml
@@ -1,78 +0,0 @@
-name: Run Terraform plan in staging
-
-on:
-  pull_request:
-    branches: [ main ]
-    paths: [ 'terraform/**' ]
-
-defaults:
-  run:
-    working-directory: terraform/staging
-
-jobs:
-  terraform:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Terraform format
-        id: format
-        run: terraform fmt -check
-
-      - name: Terraform init
-        id: init
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-
-      - name: Terraform validate
-        id: validation
-        run: terraform validate -no-color
-
-      - name: Terraform plan
-        id: plan
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
-
-      - name: Read Terraform plan output file
-        id: terraform_output
-        uses: juliangruber/read-file-action@v1
-        if: ${{ always() }}
-        with:
-          path: ./terraform/staging/plan_output.txt
-
-      # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
-      - name: Update PR
-        uses: actions/github-script@v4
-        # we would like to update the PR even when a prior step failed
-        if: ${{ always() }}
-        with:
-          script: |
-            const output = `Terraform Format and Style: ${{ steps.format.outcome }}
-            Terraform Initialization: ${{ steps.init.outcome }}
-            Terraform Validation: ${{ steps.validation.outcome }}
-            Terraform Plan: ${{ steps.plan.outcome }}
-
-            <details><summary>Show Plan</summary>
-
-            \`\`\`\n
-            ${{ steps.terraform_output.outputs.content }}
-            \`\`\`
-
-            </details>
-
-            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
-
-            github.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: output
-            })
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,6 @@ queues.csv
 __pycache__/
 *.py[cod]

-.venv/
 venv/
 venv-freeze/

@@ -70,22 +69,10 @@ target/
 *.DS_Store
 environment.sh
 .envrc
-.env
-.env*
-varsfile

 celerybeat-schedule

 # CloudFoundry
 .cf
-varsfile*
-.secret*

 /scripts/run_my_tests.sh
-
-# Terraform
-.terraform.lock.hcl
-**/.terraform/*
-secrets.auto.tfvars
-terraform.tfstate
-terraform.tfstate.backup
--- a/.pyup.yml
+++ b/.pyup.yml
@@ -4,5 +4,5 @@ schedule: "every week on wednesday"

 search: False
 requirements:
-  - requirements.in
-  - requirements_for_test.txt
+  - requirements-app.txt
+  - requirements-dev.txt
--- a/159
+++ b/159
@@ -7,52 +7,28 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)

+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
+
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
+
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
+
 CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)

+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
+
+NOTIFY_CREDENTIALS ?= ~/.notify-credentials
+

 ## DEVELOPMENT

-.PHONY: bootstrap
-bootstrap: generate-version-file ## Set up everything to run the app
-	pip3 install -r requirements_for_test.txt
-	createdb notification_api || true
-	(flask db upgrade) || true
-
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
-
-.PHONY: run-flask
-run-flask: ## Run flask
-	flask run -p 6011 --host=0.0.0.0
-
-.PHONY: run-celery
-run-celery: ## Run celery, TODO remove purge for staging/prod
-	celery -A run_celery.notify_celery purge -f
-	celery \
-		-A run_celery.notify_celery worker \
-		--pidfile="/tmp/celery.pid" \
-		--loglevel=INFO \
-		--concurrency=4
-
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
-.PHONY: run-celery-beat
-run-celery-beat: ## Run celery beat
-	celery \
-	-A run_celery.notify_celery beat \
-	--loglevel=INFO
-
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -62,29 +38,71 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}

 .PHONY: test
-test: ## Run tests
-	# flake8 .
-	isort --check-only ./app ./tests
-	pytest -n4 --maxfail=10
+test: generate-version-file ## Run tests
+	./scripts/run_tests.sh

 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pip install --upgrade pip-tools
-	pip-compile requirements.in
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze

-.PHONY: audit
-audit:
-	pip install --upgrade pip-audit
-	pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }

-.PHONY: static-scan
-static-scan:
-	pip install bandit
-	bandit -r app/
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
+	docker build -f docker/Dockerfile \
+		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
+		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
+		--build-arg NO_PROXY="${NO_PROXY}" \
+		-t ${DOCKER_BUILDER_IMAGE_NAME} \
+		.
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	@docker run -it --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
+		-e GIT_COMMIT=${GIT_COMMIT} \
+		-e BUILD_NUMBER=${BUILD_NUMBER} \
+		-e BUILD_URL=${BUILD_URL} \
+		-e http_proxy="${HTTP_PROXY}" \
+		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e https_proxy="${HTTPS_PROXY}" \
+		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e NO_PROXY="${NO_PROXY}" \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: create-docker-test-db
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	@docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true

 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
+	rm -rf node_modules cache target venv .coverage build tests/.cache


 ## DEPLOYMENT
@@ -115,6 +133,19 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"

+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@jinja2 --strict manifest.yml.j2 \
+	    -D environment=${CF_SPACE} \
+	    -D CF_APP=${CF_APP} \
+	    --format=yaml \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
+
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
@@ -123,38 +154,34 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 	@cf app --guid ${CF_APP} || exit 1

 	# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
-	cf cancel-deployment ${CF_APP} || true
+	cf v3-cancel-zdt-push ${CF_APP} || true
+
+	cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
+	CF_STARTUP_TIMEOUT=15 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete  # fails after 15 mins if deploy doesn't work

-	# fails after 15 mins if deploy doesn't work
-	CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling

 .PHONY: cf-deploy-api-db-migration
 cf-deploy-api-db-migration:
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
-	make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH}
-
-	cf push notifications-api --no-route -f ${CF_MANIFEST_PATH}
-	rm ${CF_MANIFEST_PATH}
-
-	cf run-task notifications-api --command="flask db upgrade" --name api_db_migration
+	cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
+	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration

 .PHONY: cf-check-api-db-migration-task
-cf-check-api-db-migration-task: ## Get the status for the last notifications-api task
-	@cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state"
+cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
+	@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"

 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
 	$(if ${CF_APP},,$(error Must specify CF_APP))
-	rm ${CF_MANIFEST_PATH}
-	cf cancel-deployment ${CF_APP}
+	cf v3-cancel-zdt-push ${CF_APP}

 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:
 	@echo $(shell python3 scripts/check_if_new_migration.py)

 .PHONY: cf-deploy-failwhale
-cf-deploy-failwhale:
+cf-deploy-failwhale:  #
 	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
 	cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml

--- a/3
+++ b/3
@@ -1,2 +1 @@
-web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
-worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null
+web: ./scripts/paas_app_wrapper.sh
--- a/README.md
+++ b/README.md
@@ -1,153 +1,158 @@
-# US Notify API
-
-Cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
+# GOV.UK Notify API

 Contains:
-
- the public-facing REST API for US Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation) [DOCS ARE STILL UK]
- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/18F/notifications-admin) talks to)
+- the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation)
+- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to)
 - asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc

-## QUICKSTART
---
-If you are the first on your team to deploy, set up AWS SES/SNS as instructed in the AWS setup section below.
-
-Create .env file as described in the .env section below.
-
-Install VS Code
-Open VS Code and install the Remote-Containers plug-in from Microsoft.
-
-Make sure your docker daemon is running (on OS X, this is typically accomplished by opening the Docker Desktop app)
-Also make sure there is NOT a Postgres daemon running on port 5432.
-
-Create the external docker network:
-
-`docker network create notify-network`
-
-Using the command palette (shift+cmd+p), search and select “Remote Containers: Open Folder in Container...”
-When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one).
-
-After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes.
-
-Select View->Open View..., then search/select “ports”. Await a green dot on the port view, then open a new terminal and run the web server:
-`make run-flask`
-
-Open another terminal and run the background tasks:
-`make run-celery`
-
-Confirm that everything is working by hitting localhost:6011 and it responds with a 200 OK.
-
---
 ## Setting Up

-### `.env` file
+### Python version

-Create and edit a .env file, based on sample.env.
+At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.

-NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
+### AWS credentials

-Things to change:
+To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.

- If you're not the first to deploy, only replace the aws creds, get these from team lead
- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project)
- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values
- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars
+Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.

-### AWS Setup
-
-**Steps to prepare SES**
-
-1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete.
-2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above.
-
-TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures
-
-**Steps to prepare SNS**
-
-1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms"
-2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save
-3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers"
-4. Under "Number Settings" choose "Request Phone Number"
-5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request"
-6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)"
-7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number)
-
-At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
-
-### Secrets Detection
+### Virtualenv

 ```
-brew install detect-secrets # or pip install detect-secrets
-detect-secrets scan
-#review output of above, make sure none of the baseline entries are sensitive
-detect-secrets scan > .secrets.baseline
-#creates the baseline file
+mkvirtualenv -p /usr/local/bin/python3 notifications-api
 ```

-Ideally, you'll install `detect-secrets` so that it's accessible from any environment from which you _might_ commit. You can use `brew install` to make it available globally. You could also install via `pip install` inside a virtual environment, if you're sure you'll _only_ commit from that environment.
+### `environment.sh`

-If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test).
+Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).

-You can permit exceptions by adding an inline comment containing `pragma: allowlist secret`
+Create a local environment.sh file containing the following:

-The command that is actually run by the pre-commit hook is: `git diff --staged --name-only -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+```
+echo "
+export NOTIFY_ENVIRONMENT='development'

-You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline`
+export MMG_API_KEY='MMG_API_KEY'
+export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
+export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
+
+export FLASK_APP=application.py
+export FLASK_DEBUG=1
+export WERKZEUG_DEBUG_PIN=off
+"> environment.sh
+```
+
+NOTES:
+
+ * Replace the placeholder key and prefix values as appropriate
+ * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
+ * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.

 ### Postgres

-Local postgres implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
+
+Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.

 ### Redis

-Local redis implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml)
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:

-## To test the application
+        REDIS_ENABLED = True
+
+
+##  To run the application
+
+First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
+
+You need to run the api application and a local celery instance.
+
+There are two run scripts for running all the necessary parts.

 ```
-# install dependencies, etc.
-make bootstrap
+scripts/run_app.sh
+```

+```
+scripts/run_celery.sh
+```
+
+Optionally you can also run this script to run the scheduled tasks:
+
+```
+scripts/run_celery_beat.sh
+```
+
+
+##  To test the application
+
+First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
+
+Then simply run
+
+```
 make test
 ```

-## To run a local OWASP scan
+That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
+[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.

-1. Run `make run-flask` from within the dev container.
-2. On your host machine run:
+
+## To update application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with

 ```
-docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf
+make freeze-requirements
 ```

-## To run scheduled tasks
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.

-```
-# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat
-make run-celery-beat
-```

-## To run one off tasks (Ignore for Quick Start)
+## To run one off tasks

 Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
 care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
 example, to purge all dynamically generated functional test data, do the following:

-Local (from inside the devcontainer)
-
+Locally
 ```
 flask command purge_functional_test_data -u <functional tests user name prefix>
 ```

-Remote
-
+On the server
 ```
 cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
 ```

 All commands and command options have a --help command if you need more information.

-## Further documentation [DEPRECATED]

- [Writing public APIs](docs/writing-public-apis.md)
- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
+## To create a new worker app
+
+You need to:
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
+1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
+1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
+1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
+
+**Important:**
+
+Before pushing the deployment change on jenkins, read below about the first time deployment.
+
+### First time deployment of your new worker
+
+Our deployment flow requires that the app is present in order to proceed with the deployment.
+
+This means that the first deployment of your app must happen manually.
+
+To do this:
+
+1. Ensure your code is backwards compatible
+1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
+
+Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.
--- a/app/init.py
+++ b/app/init.py
@@ -1,74 +1,74 @@
-import os
-import secrets
-import string
 import time
+import os
+import random
+import string
 import uuid
-from time import monotonic

 from celery import current_task
-from flask import (
-    current_app,
-    g,
-    has_request_context,
-    jsonify,
-    make_response,
-    request,
-)
+from flask import _request_ctx_stack, request, g, jsonify, make_response, current_app, has_request_context
+from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from gds_metrics import GDSMetrics
 from gds_metrics.metrics import Gauge, Histogram
-from notifications_utils import logging, request_helper
-from notifications_utils.celery import NotifyCelery
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from time import monotonic
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
+from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils import logging, request_helper
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy

-from app.clients import NotificationProviderClients
-from app.clients.cbc_proxy import CBCProxyClient
+from app.celery.celery import NotifyCelery
+from app.clients import Clients
+from app.clients.cbc_proxy import CBCProxyClient, CBCProxyNoopClient
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.aws_sns import AwsSnsClient
+from app.clients.sms.firetext import FiretextClient
+from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+
+DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+DATE_FORMAT = "%Y-%m-%d"


 class SQLAlchemy(_SQLAlchemy):
    """We need to subclass SQLAlchemy in order to override create_engine options"""

    def apply_driver_hacks(self, app, info, options):
-        sa_url, options = super().apply_driver_hacks(app, info, options)
+        super().apply_driver_hacks(app, info, options)
        if 'connect_args' not in options:
            options['connect_args'] = {}
        options['connect_args']["options"] = "-c statement_timeout={}".format(
            int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
        )
-        return (sa_url, options)


 db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
+firetext_client = FiretextClient()
+mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 aws_ses_stub_client = AwsSesStubClient()
-aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
-cbc_proxy_client = CBCProxyClient()
+performance_platform_client = PerformancePlatformClient()
+cbc_proxy_client = CBCProxyNoopClient()
 document_download_client = DocumentDownloadClient()
 metrics = GDSMetrics()

-notification_provider_clients = NotificationProviderClients()
+clients = Clients()

-api_user = LocalProxy(lambda: g.api_user)
-authenticated_service = LocalProxy(lambda: g.authenticated_service)
+api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
+authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)

 CONCURRENT_REQUESTS = Gauge(
    'concurrent_web_request_count',
@@ -95,7 +95,8 @@ def create_app(application):
    zendesk_client.init_app(application)
    statsd_client.init_app(application)
    logging.init_app(application, statsd_client)
-    aws_sns_client.init_app(application, statsd_client=statsd_client)
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application, statsd_client=statsd_client)

    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
    aws_ses_stub_client.init_app(
@@ -105,16 +106,17 @@ def create_app(application):
    )
    # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
    email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
-    notification_provider_clients.init_app(
-        sms_clients=[aws_sns_client],
-        email_clients=email_clients
-    )
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients)

    notify_celery.init_app(application)
    encryption.init_app(application)
    redis_store.init_app(application)
+    performance_platform_client.init_app(application)
    document_download_client.init_app(application)

+    global cbc_proxy_client
+    if application.config['CBC_PROXY_AWS_ACCESS_KEY_ID']:
+        cbc_proxy_client = CBCProxyClient()
    cbc_proxy_client.init_app(application)

    register_blueprint(application)
@@ -131,59 +133,35 @@ def create_app(application):


 def register_blueprint(application):
-    from app.authentication.auth import (
-        requires_admin_auth,
-        requires_auth,
-        requires_govuk_alerts_auth,
-        requires_no_auth,
-    )
-    from app.billing.rest import billing_blueprint
-    from app.broadcast_message.rest import broadcast_message_blueprint
-    from app.complaint.complaint_rest import complaint_blueprint
-    from app.email_branding.rest import email_branding_blueprint
+    from app.service.rest import service_blueprint
+    from app.service.callback_rest import service_callback_blueprint
+    from app.user.rest import user_blueprint
+    from app.template.rest import template_blueprint
+    from app.status.healthcheck import status as status_blueprint
+    from app.job.rest import job_blueprint
+    from app.notifications.rest import notifications as notifications_blueprint
+    from app.invite.rest import invite as invite_blueprint
+    from app.accept_invite.rest import accept_invite
+    from app.template_statistics.rest import template_statistics as template_statistics_blueprint
    from app.events.rest import events as events_blueprint
-    from app.govuk_alerts.rest import govuk_alerts_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.email_branding.rest import email_branding_blueprint
    from app.inbound_number.rest import inbound_number_blueprint
    from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
-    from app.job.rest import job_blueprint
-    from app.letter_branding.letter_branding_rest import (
-        letter_branding_blueprint,
-    )
+    from app.notifications.receive_notifications import receive_notifications_blueprint
+    from app.notifications.notifications_sms_callback import sms_callback_blueprint
+    from app.notifications.notifications_letter_callback import letter_callback_blueprint
+    from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
    from app.letters.rest import letter_job
-    from app.notifications.notifications_letter_callback import (
-        letter_callback_blueprint,
-    )
-    from app.notifications.notifications_ses_callback import (
-        ses_callback_blueprint,
-    )
-    from app.notifications.notifications_sms_callback import (
-        sms_callback_blueprint,
-    )
-    from app.notifications.receive_notifications import (
-        receive_notifications_blueprint,
-    )
-    from app.notifications.rest import notifications as notifications_blueprint
-    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.billing.rest import billing_blueprint
    from app.organisation.rest import organisation_blueprint
-    from app.performance_dashboard.rest import performance_dashboard_blueprint
+    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.complaint.complaint_rest import complaint_blueprint
    from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import (
-        provider_details as provider_details_blueprint,
-    )
-    from app.service.callback_rest import service_callback_blueprint
-    from app.service.rest import service_blueprint
-    from app.service_invite.rest import (
-        service_invite as service_invite_blueprint,
-    )
-    from app.status.healthcheck import status as status_blueprint
-    from app.template.rest import template_blueprint
    from app.template_folder.rest import template_folder_blueprint
-    from app.template_statistics.rest import (
-        template_statistics as template_statistics_blueprint,
-    )
+    from app.letter_branding.letter_branding_rest import letter_branding_blueprint
    from app.upload.rest import upload_blueprint
-    from app.user.rest import user_blueprint
-    from app.webauthn.rest import webauthn_blueprint
+    from app.broadcast_message.rest import broadcast_message_blueprint

    service_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -191,18 +169,11 @@ def register_blueprint(application):
    user_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(user_blueprint, url_prefix='/user')

-    webauthn_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(webauthn_blueprint)
-
    template_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(template_blueprint)

    status_blueprint.before_request(requires_no_auth)
    application.register_blueprint(status_blueprint)
-    
-    # delivery receipts
-    ses_callback_blueprint.before_request(requires_no_auth)
-    application.register_blueprint(ses_callback_blueprint)

    # delivery receipts
    # TODO: make sure research mode can still trigger sms callbacks, then re-enable this
@@ -219,11 +190,8 @@ def register_blueprint(application):
    job_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(job_blueprint)

-    service_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_invite_blueprint)
-
-    organisation_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organisation_invite_blueprint)
+    invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(invite_blueprint)

    inbound_number_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(inbound_number_blueprint)
@@ -231,6 +199,9 @@ def register_blueprint(application):
    inbound_sms_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(inbound_sms_blueprint)

+    accept_invite.before_request(requires_admin_auth)
+    application.register_blueprint(accept_invite, url_prefix='/invite')
+
    template_statistics_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(template_statistics_blueprint)

@@ -258,12 +229,12 @@ def register_blueprint(application):
    organisation_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(organisation_blueprint, url_prefix='/organisations')

+    organisation_invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(organisation_invite_blueprint)
+
    complaint_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(complaint_blueprint)

-    performance_dashboard_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(performance_dashboard_blueprint)
-
    platform_stats_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')

@@ -279,40 +250,33 @@ def register_blueprint(application):
    broadcast_message_blueprint.before_request(requires_admin_auth)
    application.register_blueprint(broadcast_message_blueprint)

-    govuk_alerts_blueprint.before_request(requires_govuk_alerts_auth)
-    application.register_blueprint(govuk_alerts_blueprint)
-

 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
+    from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
+    from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
+    from app.v2.template.get_template import v2_template_blueprint as get_template
+    from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
+    from app.v2.template.post_template import v2_template_blueprint as post_template
    from app.authentication.auth import requires_auth
-    from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint

-    v2_notification_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
+    post_notifications.before_request(requires_auth)
+    application.register_blueprint(post_notifications)

-    v2_templates_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
+    get_notifications.before_request(requires_auth)
+    application.register_blueprint(get_notifications)

-    v2_template_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
+    get_templates.before_request(requires_auth)
+    application.register_blueprint(get_templates)

-    v2_inbound_sms_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
+    get_template.before_request(requires_auth)
+    application.register_blueprint(get_template)

-    v2_broadcast_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_broadcast_blueprint)
+    post_template.before_request(requires_auth)
+    application.register_blueprint(post_template)
+
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)


 def init_app(app):
@@ -360,7 +324,7 @@ def create_uuid():


 def create_random_identifier():
-    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))


 def setup_sqlalchemy_events(app):
@@ -420,7 +384,7 @@ def setup_sqlalchemy_events(app):
                        'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
                        'url_rule': current_task.name,  # task name
                    }
-                # anything else. migrations possibly, or flask cli commands.
+                # anything else. migrations possibly.
                else:
                    current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
                    connection_record.info['request_data'] = {
--- a/app/accept_invite/init.py
+++ b/app/accept_invite/init.py
--- a/app/accept_invite/rest.py
+++ b/app/accept_invite/rest.py
@@ -0,0 +1,52 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    current_app
+)
+
+from itsdangerous import SignatureExpired, BadData
+
+from notifications_utils.url_safe_token import check_token
+
+from app.dao.invited_user_dao import get_invited_user_by_id
+from app.dao.organisation_dao import dao_get_invited_organisation_user
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
+from app.schemas import invited_user_schema
+
+
+accept_invite = Blueprint('accept_invite', __name__)
+register_errors(accept_invite)
+
+
+@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
+def validate_invitation_token(invitation_type, token):
+
+    max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
+
+    try:
+        invited_user_id = check_token(token,
+                                      current_app.config['SECRET_KEY'],
+                                      current_app.config['DANGEROUS_SALT'],
+                                      max_age_seconds)
+    except SignatureExpired:
+        errors = {'invitation':
+                  'Your invitation to GOV.UK Notify has expired. '
+                  'Please ask the person that invited you to send you another one'}
+        raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
+
+    if invitation_type == 'service':
+        invited_user = get_invited_user_by_id(invited_user_id)
+        return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+    elif invitation_type == 'organisation':
+        invited_user = dao_get_invited_organisation_user(invited_user_id)
+        return jsonify(data=invited_user.serialize()), 200
+    else:
+        raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))
--- a/app/authentication/auth.py
+++ b/app/authentication/auth.py
@@ -1,24 +1,17 @@
-import uuid
-
-from flask import current_app, g, request
-from gds_metrics import Histogram
-from notifications_python_client.authentication import (
-    decode_jwt_token,
-    get_token_issuer,
-)
+from flask import request, _request_ctx_stack, current_app, g
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
 from notifications_python_client.errors import (
-    TokenAlgorithmError,
-    TokenDecodeError,
-    TokenError,
-    TokenExpiredError,
-    TokenIssuerError,
+    TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError, TokenError
 )
 from notifications_utils import request_helper
+from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
+from gds_metrics import Histogram

 from app.serialised_models import SerialisedService

-GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # nosec B105
+
+GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa

 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
    'auth_db_connection_duration_seconds',
@@ -49,115 +42,7 @@ class AuthError(Exception):
        }


-class InternalApiKey():
-    def __init__(self, client_id, secret):
-        self.secret = secret
-        self.id = client_id
-        self.expiry_date = None
-
-
-def requires_no_auth():
-    pass
-
-
-def requires_govuk_alerts_auth():
-    requires_internal_auth(current_app.config.get('GOVUK_ALERTS_CLIENT_ID'))
-
-
-def requires_admin_auth():
-    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
-
-
-def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
-        raise TypeError("Unknown client_id for internal auth")
-
-    request_helper.check_proxy_header_before_request()
-    auth_token = _get_auth_token(request)
-    client_id = _get_token_issuer(auth_token)
-
-    if client_id != expected_client_id:
-        current_app.logger.info('client_id: %s', client_id)
-        current_app.logger.info('expected_client_id: %s', expected_client_id)
-        raise AuthError("Unauthorized: not allowed to perform this action", 401)
-
-    api_keys = [
-        InternalApiKey(client_id, secret)
-        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
-    ]
-
-    _decode_jwt_token(auth_token, api_keys, client_id)
-    g.service_id = client_id
-
-
-def requires_auth():
-    request_helper.check_proxy_header_before_request()
-
-    auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
-
-    try:
-        service_id = uuid.UUID(issuer)
-    except Exception:
-        raise AuthError("Invalid token: service id is not the right data type", 403)
-
-    try:
-        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
-            service = SerialisedService.from_id(service_id)
-    except NoResultFound:
-        raise AuthError("Invalid token: service not found", 403)
-
-    if not service.api_keys:
-        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
-
-    if not service.active:
-        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
-
-    api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
-
-    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
-        service_id,
-        api_key.id,
-        request.headers.get('User-Agent'),
-        request.base_url
-    ))
-
-    g.api_user = api_key
-    g.service_id = service_id
-    g.authenticated_service = service
-
-
-def _decode_jwt_token(auth_token, api_keys, service_id=None):
-    for api_key in api_keys:
-        try:
-            decode_jwt_token(auth_token, api_key.secret)
-        except TokenExpiredError:
-            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenAlgorithmError:
-            err_msg = "Invalid token: algorithm used is not HS256"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenDecodeError:
-            # we attempted to validate the token but it failed meaning it was not signed using this api key.
-            # Let's try the next one
-            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
-            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
-            # continue on to check the next API key
-            continue
-        except TokenError:
-            # General error when trying to decode and validate the token
-            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
-
-        if api_key.expiry_date:
-            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
-
-        return api_key
-    else:
-        # service has API keys, but none matching the one the user provided
-        raise AuthError("Invalid token: API key not found", 403, service_id=service_id)
-
-
-def _get_auth_token(req):
+def get_auth_token(req):
    auth_header = req.headers.get('Authorization', None)
    if not auth_header:
        raise AuthError('Unauthorized: authentication token must be provided', 401)
@@ -170,7 +55,97 @@ def _get_auth_token(req):
    return auth_header[7:]


-def _get_token_issuer(auth_token):
+def requires_no_auth():
+    pass
+
+
+def requires_admin_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
+
+    if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
+        g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
+
+        for secret in current_app.config.get('API_INTERNAL_SECRETS'):
+            try:
+                decode_jwt_token(auth_token, secret)
+                return
+            except TokenExpiredError:
+                raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
+            except TokenDecodeError:
+                # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions
+                # (which are children of `TokenDecodeError`) as these should cause an auth error immediately rather
+                # than continue on to check the next admin client secret
+                continue
+
+        # Either there are no admin client secrets or their token didn't match one of them so error
+        raise AuthError("Unauthorized: admin authentication token not found", 401)
+    else:
+        raise AuthError('Unauthorized: admin authentication token required', 401)
+
+
+def requires_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    issuer = __get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
+
+    try:
+        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
+            service = SerialisedService.from_id(issuer)
+    except DataError:
+        raise AuthError("Invalid token: service id is not the right data type", 403)
+    except NoResultFound:
+        raise AuthError("Invalid token: service not found", 403)
+
+    if not service.api_keys:
+        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
+
+    if not service.active:
+        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
+
+    for api_key in service.api_keys:
+        try:
+            decode_jwt_token(auth_token, api_key.secret)
+        except TokenExpiredError:
+            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenAlgorithmError:
+            err_msg = "Invalid token: algorithm used is not HS256"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenDecodeError:
+            # we attempted to validate the token but it failed meaning it was not signed using this api key.
+            # Let's try the next one
+            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
+            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
+            # continue on to check the next API key
+            continue
+        except TokenError:
+            # General error when trying to decode and validate the token
+            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service.id, api_key_id=api_key.id)
+
+        if api_key.expiry_date:
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
+
+        g.service_id = service.id
+        _request_ctx_stack.top.authenticated_service = service
+        _request_ctx_stack.top.api_user = api_key
+
+        current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
+            service.id,
+            api_key.id,
+            request.headers.get('User-Agent'),
+            request.base_url
+        ))
+        return
+    else:
+        # service has API keys, but none matching the one the user provided
+        raise AuthError("Invalid token: API key not found", 403, service_id=service.id)
+
+
+def __get_token_issuer(auth_token):
    try:
        issuer = get_token_issuer(auth_token)
    except TokenIssuerError:
--- a/app/aws/s3.py
+++ b/app/aws/s3.py
@@ -1,30 +1,31 @@
-import os
-
-import botocore
-from boto3 import Session, client
 from flask import current_app

+from boto3 import client, resource
+import botocore
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'

-default_access_key = os.environ.get('AWS_ACCESS_KEY_ID')
-default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
-default_region = os.environ.get('AWS_REGION')

-def get_s3_file(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
-    s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
+def get_s3_file(bucket_name, file_location):
+    s3_file = get_s3_object(bucket_name, file_location)
    return s3_file.get()['Body'].read().decode('utf-8')


-def get_s3_object(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
-    session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
-    s3 = session.resource('s3')
+def get_s3_object(bucket_name, file_location):
+    s3 = resource('s3')
    return s3.Object(bucket_name, file_location)


-def file_exists(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region):
+def head_s3_object(bucket_name, file_location):
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    return boto_client.head_object(Bucket=bucket_name, Key=file_location)
+
+
+def file_exists(bucket_name, file_location):
    try:
        # try and access metadata of object
-        get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata
+        get_s3_object(bucket_name, file_location).metadata
        return True
    except botocore.exceptions.ClientError as e:
        if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
@@ -36,9 +37,6 @@ def get_job_location(service_id, job_id):
    return (
        current_app.config['CSV_UPLOAD_BUCKET_NAME'],
        FILE_LOCATION_STRUCTURE.format(service_id, job_id),
-        current_app.config['CSV_UPLOAD_ACCESS_KEY'],
-        current_app.config['CSV_UPLOAD_SECRET_KEY'],
-        current_app.config['CSV_UPLOAD_REGION'],
    )


@@ -46,9 +44,6 @@ def get_contact_list_location(service_id, contact_list_id):
    return (
        current_app.config['CONTACT_LIST_BUCKET_NAME'],
        FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
-        current_app.config['CONTACT_LIST_ACCESS_KEY'],
-        current_app.config['CONTACT_LIST_SECRET_KEY'],
-        current_app.config['CONTACT_LIST_REGION'],
    )


@@ -75,21 +70,29 @@ def remove_contact_list_from_s3(service_id, contact_list_id):
    return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))


-def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
-    obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
+def get_s3_bucket_objects(bucket_name, subfolder=''):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        if page.get('Contents'):
+            all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
+
+
+def remove_s3_object(bucket_name, object_key):
+    obj = get_s3_object(bucket_name, object_key)
    return obj.delete()


-def get_list_of_files_by_suffix(
-    bucket_name,
-    subfolder='',
-    suffix='',
-    last_modified=None,
-    access_key=default_access_key,
-    secret_key=default_secret_key,
-    region=default_region
-):
-    s3_client = client('s3', region, aws_access_key_id=access_key, aws_secret_access_key=secret_key)
+def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None):
+    s3_client = client('s3', current_app.config['AWS_REGION'])
    paginator = s3_client.get_paginator('list_objects_v2')

    page_iterator = paginator.paginate(
--- a/app/billing/billing_schemas.py
+++ b/app/billing/billing_schemas.py
@@ -1,45 +1,42 @@
 from datetime import datetime

+
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST annual billing schema",
    "type": "object",
    "title": "Create",
    "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
    },
    "required": ["free_sms_fragment_limit"]
 }


-def serialize_ft_billing_remove_emails(rows):
-    return [
-        {
-            "month": (datetime.strftime(row.month, "%B")),
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "postage": row.postage,
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_remove_emails(data):
+    results = []
+    billed_notifications = [x for x in data if x.notification_type != 'email']
+    for notification in billed_notifications:
+        json_result = {
+            "month": (datetime.strftime(notification.month, "%B")),
+            "notification_type": notification.notification_type,
+            "billing_units": notification.billable_units,
+            "rate": float(notification.rate),
+            "postage": notification.postage,
        }
-        for row in rows
-        if row.notification_type != 'email'
-    ]
+        results.append(json_result)
+    return results


-def serialize_ft_billing_yearly_totals(rows):
-    return [
-        {
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
        }
-        for row in rows
-    ]
+        yearly_totals.append(json_result)
+
+    return yearly_totals
--- a/app/billing/rest.py
+++ b/app/billing/rest.py
@@ -6,18 +6,18 @@ from app.billing.billing_schemas import (
    serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
    dao_get_free_sms_fragment_limit_for_year,
-    dao_update_annual_billing_for_future_years,
-    set_default_free_allowance_for_service,
+    dao_get_all_free_sms_fragment_limit,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
-    fetch_billing_totals_for_year,
-    fetch_monthly_billing_for_year,
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
 )
+
+from app.errors import InvalidRequest
 from app.errors import register_errors
-from app.models import Service
 from app.schema_validation import validate

 billing_blueprint = Blueprint(
@@ -30,6 +30,7 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)


+@billing_blueprint.route('/ft-monthly-usage')
@billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
    try:
@@ -41,6 +42,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
    return jsonify(data)


+@billing_blueprint.route('/ft-yearly-usage-summary')
@billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
    try:
@@ -61,14 +63,27 @@ def get_free_sms_fragment_limit(service_id):
    annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)

    if annual_billing is None:
-        service = Service.query.get(service_id)
-        # An entry does not exist in annual_billing table for that service and year.
-        # Set the annual billing to the default free allowance based on the organisation type of the service.
+        # An entry does not exist in annual_billing table for that service and year. If it is a past year,
+        # we return the oldest entry.
+        # If it is the current or future years, we create an entry in the db table using the newest record,
+        # and return that number.  If all fails, we return InvalidRequest.
+        sms_list = dao_get_all_free_sms_fragment_limit(service_id)

-        annual_billing = set_default_free_allowance_for_service(
-            service=service,
-            year_start=int(financial_year_start) if financial_year_start else None
-        )
+        if not sms_list:
+            raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
+        else:
+            if financial_year_start is None:
+                financial_year_start = get_current_financial_year_start_year()
+
+            if int(financial_year_start) < get_current_financial_year_start_year():
+                # return the earliest historical entry
+                annual_billing = sms_list[0]   # The oldest entry
+            else:
+                annual_billing = sms_list[-1]  # The newest entry
+
+                annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
+                                                                              annual_billing.free_sms_fragment_limit,
+                                                                              financial_year_start)

    return jsonify(annual_billing.serialize_free_sms_items()), 200

--- a/app/broadcast_message/broadcast_message_schema.py
+++ b/app/broadcast_message/broadcast_message_schema.py
@@ -1,8 +1,8 @@
-from app.models import BroadcastStatusType
 from app.schema_validation.definitions import uuid
+from app.models import BroadcastStatusType

 create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
    'description': 'POST create broadcast_message schema',
    'type': 'object',
    'title': 'Create broadcast_message',
@@ -13,26 +13,15 @@ create_broadcast_message_schema = {
        'personalisation': {'type': 'object'},
        'starts_at': {'type': 'string', 'format': 'datetime'},
        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-        'content': {'type': 'string', 'minLength': 1},
-        'reference': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'areas': {"type": "array", "items": {"type": "string"}},
+        'simple_polygons': {"type": "array", "items": {"type": "array"}},
    },
-    'required': ['service_id', 'created_by'],
-    'allOf': [
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['content']},
-        ]},
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['reference']},
-        ]},
-    ],
+    'required': ['template_id', 'service_id', 'created_by'],
    'additionalProperties': False
 }

 update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
    'description': 'POST update broadcast_message schema',
    'type': 'object',
    'title': 'Update broadcast_message',
@@ -40,14 +29,15 @@ update_broadcast_message_schema = {
        'personalisation': {'type': 'object'},
        'starts_at': {'type': 'string', 'format': 'datetime'},
        'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
+        'areas': {"type": "array", "items": {"type": "string"}},
+        'simple_polygons': {"type": "array", "items": {"type": "array"}},
    },
    'required': [],
    'additionalProperties': False
 }

 update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
    'description': 'POST update broadcast_message status schema',
    'type': 'object',
    'title': 'Update broadcast_message',
--- a/app/broadcast_message/rest.py
+++ b/app/broadcast_message/rest.py
@@ -1,23 +1,24 @@
-import iso8601
-from flask import Blueprint, jsonify, request
-from notifications_utils.template import BroadcastMessageTemplate
+from datetime import datetime

-from app.broadcast_message import utils as broadcast_utils
+import iso8601
+from flask import Blueprint, jsonify, request, current_app
+from app.config import QueueNames
+from app.dao.dao_utils import dao_save_object
+from app.dao.templates_dao import dao_get_template_by_id_and_service_id
+from app.dao.users_dao import get_user_by_id
+from app.dao.broadcast_message_dao import (
+    dao_get_broadcast_message_by_id_and_service_id,
+    dao_get_broadcast_messages_for_service,
+)
+from app.dao.services_dao import dao_fetch_service_by_id
+from app.errors import register_errors, InvalidRequest
+from app.models import BroadcastMessage, BroadcastStatusType, BroadcastEvent, BroadcastEventMessageType
+from app.celery.broadcast_message_tasks import send_broadcast_event
 from app.broadcast_message.broadcast_message_schema import (
    create_broadcast_message_schema,
    update_broadcast_message_schema,
    update_broadcast_message_status_schema,
 )
-from app.dao.broadcast_message_dao import (
-    dao_get_broadcast_message_by_id_and_service_id,
-    dao_get_broadcast_messages_for_service,
-)
-from app.dao.dao_utils import dao_save_object
-from app.dao.services_dao import dao_fetch_service_by_id
-from app.dao.templates_dao import dao_get_template_by_id_and_service_id
-from app.dao.users_dao import get_user_by_id
-from app.errors import InvalidRequest, register_errors
-from app.models import BroadcastMessage, BroadcastStatusType
 from app.schema_validation import validate

 broadcast_message_blueprint = Blueprint(
@@ -34,6 +35,49 @@ def _parse_nullable_datetime(dt):
    return dt


+def _update_broadcast_message(broadcast_message, new_status, updating_user):
+    if updating_user not in broadcast_message.service.users:
+        raise InvalidRequest(
+            f'User {updating_user.id} cannot approve broadcast_message {broadcast_message.id} from other service',
+            status_code=400
+        )
+
+    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
+        raise InvalidRequest(
+            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
+            status_code=400
+        )
+
+    if new_status == BroadcastStatusType.BROADCASTING:
+        # TODO: Remove this platform admin shortcut when the feature goes live
+        if updating_user == broadcast_message.created_by and not (
+            # platform admins and trial mode services can approve their own broadcasts
+            updating_user.platform_admin or
+            broadcast_message.service.restricted
+        ):
+            raise InvalidRequest(
+                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
+                status_code=400
+            )
+        elif not broadcast_message.areas:
+            raise InvalidRequest(
+                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
+                status_code=400
+            )
+        else:
+            broadcast_message.approved_at = datetime.utcnow()
+            broadcast_message.approved_by = updating_user
+
+    if new_status == BroadcastStatusType.CANCELLED:
+        broadcast_message.cancelled_at = datetime.utcnow()
+        broadcast_message.cancelled_by = updating_user
+
+    current_app.logger.info(
+        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
+    )
+    broadcast_message.status = new_status
+
+
@broadcast_message_blueprint.route('', methods=['GET'])
 def get_broadcast_messages_for_service(service_id):
    # TODO: should this return template content/data in some way? or can we rely on them being cached admin side.
@@ -55,48 +99,18 @@ def create_broadcast_message(service_id):
    validate(data, create_broadcast_message_schema)
    service = dao_fetch_service_by_id(data['service_id'])
    user = get_user_by_id(data['created_by'])
-    personalisation = data.get('personalisation', {})
-    template_id = data.get('template_id')
-
-    if template_id:
-        template = dao_get_template_by_id_and_service_id(
-            template_id, data['service_id']
-        )
-        content = str(template._as_utils_template_with_personalisation(
-            personalisation
-        ))
-        reference = None
-    else:
-        temporary_template = BroadcastMessageTemplate.from_content(data['content'])
-        if temporary_template.content_too_long:
-            raise InvalidRequest(
-                (
-                    f'Content must be '
-                    f'{temporary_template.max_content_count:,.0f} '
-                    f'characters or fewer'
-                ) + (
-                    ' (because it could not be GSM7 encoded)'
-                    if temporary_template.non_gsm_characters else ''
-                ),
-                status_code=400,
-            )
-        template = None
-        content = str(temporary_template)
-        reference = data['reference']
+    template = dao_get_template_by_id_and_service_id(data['template_id'], data['service_id'])

    broadcast_message = BroadcastMessage(
        service_id=service.id,
-        template_id=template_id,
-        template_version=template.version if template else None,
-        personalisation=personalisation,
-        areas=data.get("areas", {}),
+        template_id=template.id,
+        template_version=template.version,
+        personalisation=data.get('personalisation', {}),
+        areas={"areas": data.get("areas", []), "simple_polygons": data.get("simple_polygons", [])},
        status=BroadcastStatusType.DRAFT,
        starts_at=_parse_nullable_datetime(data.get('starts_at')),
        finishes_at=_parse_nullable_datetime(data.get('finishes_at')),
        created_by_id=user.id,
-        content=content,
-        reference=reference,
-        stubbed=service.restricted
    )

    dao_save_object(broadcast_message)
@@ -107,6 +121,7 @@ def create_broadcast_message(service_id):
@broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['POST'])
 def update_broadcast_message(service_id, broadcast_message_id):
    data = request.get_json()
+
    validate(data, update_broadcast_message_schema)

    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
@@ -117,11 +132,9 @@ def update_broadcast_message(service_id, broadcast_message_id):
            status_code=400
        )

-    areas = data.get("areas", {})
-
-    if ('ids' in areas and 'simple_polygons' not in areas) or ('ids' not in areas and 'simple_polygons' in areas):
+    if ('areas' in data and 'simple_polygons' not in data) or ('areas' not in data and 'simple_polygons' in data):
        raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.',
+            f'Cannot update broadcast_message {broadcast_message.id}, areas or polygons are missing.',
            status_code=400
        )

@@ -131,8 +144,8 @@ def update_broadcast_message(service_id, broadcast_message_id):
        broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at'])
    if 'finishes_at' in data:
        broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at'])
-    if 'ids' in areas and 'simple_polygons' in areas:
-        broadcast_message.areas = areas
+    if 'areas' in data and 'simple_polygons' in data:
+        broadcast_message.areas = {"areas": data["areas"], "simple_polygons": data["simple_polygons"]}

    dao_save_object(broadcast_message)

@@ -146,20 +159,53 @@ def update_broadcast_message_status(service_id, broadcast_message_id):
    validate(data, update_broadcast_message_status_schema)
    broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)

-    if not broadcast_message.service.active:
-        raise InvalidRequest("Updating broadcast message is not allowed: service is inactive ", 403)
-
    new_status = data['status']
    updating_user = get_user_by_id(data['created_by'])

-    if updating_user not in broadcast_message.service.users:
-        #  we allow platform admins to cancel broadcasts, and we don't check user if request was done via API
-        if not (new_status == BroadcastStatusType.CANCELLED and updating_user.platform_admin):
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot update broadcast_message {broadcast_message.id} from other service',
-                status_code=400
-            )
+    _update_broadcast_message(broadcast_message, new_status, updating_user)
+    dao_save_object(broadcast_message)

-    broadcast_utils.update_broadcast_message_status(broadcast_message, new_status, updating_user)
+    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
+        _create_broadcast_event(broadcast_message)

    return jsonify(broadcast_message.serialize()), 200
+
+
+def _create_broadcast_event(broadcast_message):
+    """
+    Creates a broadcast event, stores it in the database, and triggers the task to send the CAP XML off
+    """
+    msg_types = {
+        BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
+        BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
+    }
+
+    if broadcast_message.status == BroadcastStatusType.CANCELLED:
+        transmitted_finishes_at = broadcast_message.cancelled_at
+    else:
+        transmitted_finishes_at = broadcast_message.finishes_at
+
+    # TODO: This doesn't support placeholders yet. We shouldn't use BroadcastMessageTemplate when we add placeholders
+    # as that just outputs XML, we need the raw text.
+    event = BroadcastEvent(
+        service=broadcast_message.service,
+        broadcast_message=broadcast_message,
+        message_type=msg_types[broadcast_message.status],
+        transmitted_content={"body": broadcast_message.template.content},
+        transmitted_areas=broadcast_message.areas,
+        # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it include
+        # a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
+        transmitted_sender='notifications.service.gov.uk',
+
+        # TODO: Should this be set to now? Or the original starts_at?
+        transmitted_starts_at=broadcast_message.starts_at,
+        # TODO: When cancelling, do we need to set this to now? Or should we keep it as the original time.
+        transmitted_finishes_at=transmitted_finishes_at,
+    )
+
+    dao_save_object(event)
+
+    send_broadcast_event.apply_async(
+        kwargs={'broadcast_event_id': str(event.id)},
+        queue=QueueNames.NOTIFY
+    )
--- a/app/broadcast_message/translators.py
+++ b/app/broadcast_message/translators.py
@@ -1,37 +0,0 @@
-from bs4 import BeautifulSoup
-
-
-def cap_xml_to_dict(cap_xml):
-    # This function assumes that it’s being passed valid CAP XML
-    cap = BeautifulSoup(cap_xml, "xml")
-    return {
-        "msgType": cap.alert.msgType.text,
-        "reference": cap.alert.identifier.text,
-        "references": (
-            # references to previous events belonging to the same alert
-            cap.alert.references.text if cap.alert.references else None
-        ),
-        "cap_event": cap.alert.info.event.text,
-        "category": cap.alert.info.category.text,
-        "expires": cap.alert.info.expires.text,
-        "content": cap.alert.info.description.text,
-        "areas": [
-            {
-                "name": area.areaDesc.text,
-                "polygons": [
-                    cap_xml_polygon_to_list(polygon.text)
-                    for polygon in area.find_all('polygon')
-                ]
-            }
-            for area in cap.alert.info.find_all('area')
-        ]
-    }
-
-
-def cap_xml_polygon_to_list(polygon_string):
-    return [
-        [
-            float(coordinate) for coordinate in pair.split(',')
-        ]
-        for pair in polygon_string.strip().split(' ')
-    ]
--- a/app/broadcast_message/utils.py
+++ b/app/broadcast_message/utils.py
@@ -1,140 +0,0 @@
-import inspect
-from datetime import datetime
-
-from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-
-from app import zendesk_client
-from app.celery.broadcast_message_tasks import send_broadcast_event
-from app.config import QueueNames
-from app.dao.dao_utils import dao_save_object
-from app.errors import InvalidRequest
-from app.models import (
-    BroadcastEvent,
-    BroadcastEventMessageType,
-    BroadcastStatusType,
-)
-
-
-def update_broadcast_message_status(broadcast_message, new_status, updating_user=None, api_key_id=None):
-    _validate_broadcast_update(broadcast_message, new_status, updating_user)
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        broadcast_message.approved_at = datetime.utcnow()
-        broadcast_message.approved_by = updating_user
-
-    if new_status == BroadcastStatusType.CANCELLED:
-        broadcast_message.cancelled_at = datetime.utcnow()
-        broadcast_message.cancelled_by = updating_user
-        broadcast_message.cancelled_by_api_key_id = api_key_id
-
-    current_app.logger.info(
-        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
-    )
-    broadcast_message.status = new_status
-
-    dao_save_object(broadcast_message)
-    _create_p1_zendesk_alert(broadcast_message)
-
-    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
-        _create_broadcast_event(broadcast_message)
-
-
-def _validate_broadcast_update(broadcast_message, new_status, updating_user):
-    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
-        raise InvalidRequest(
-            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
-            status_code=400
-        )
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        # training mode services can approve their own broadcasts
-        if updating_user == broadcast_message.created_by and not broadcast_message.service.restricted:
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
-                status_code=400
-            )
-        elif len(broadcast_message.areas['simple_polygons']) == 0:
-            raise InvalidRequest(
-                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
-                status_code=400
-            )
-
-
-def _create_p1_zendesk_alert(broadcast_message):
-    if current_app.config['NOTIFY_ENVIRONMENT'] != 'live':
-        return
-
-    if broadcast_message.status != BroadcastStatusType.BROADCASTING:
-        return
-
-    message = inspect.cleandoc(f"""
-        Broadcast Sent
-
-        https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}
-
-        Sent on channel {broadcast_message.service.broadcast_channel} to {broadcast_message.areas["names"]}.
-
-        Content starts "{broadcast_message.content[:100]}".
-
-        Follow the runbook to check the broadcast went out OK:
-        https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs/edit#heading=h.lzr9aq5b4wg
-    """)
-
-    ticket = NotifySupportTicket(
-        subject='Live broadcast sent',
-        message=message,
-        ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-        technical_ticket=True,
-        org_id=current_app.config['BROADCAST_ORGANISATION_ID'],
-        org_type='federal',
-        service_id=str(broadcast_message.service_id),
-        p1=True
-    )
-    zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-def _create_broadcast_event(broadcast_message):
-    """
-    If the service is live and the broadcast message is not stubbed, creates a broadcast event, stores it in the
-    database, and triggers the task to send the CAP XML off.
-    """
-    service = broadcast_message.service
-
-    if not broadcast_message.stubbed and not service.restricted:
-        msg_types = {
-            BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
-            BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
-        }
-
-        event = BroadcastEvent(
-            service=service,
-            broadcast_message=broadcast_message,
-            message_type=msg_types[broadcast_message.status],
-            transmitted_content={"body": broadcast_message.content},
-            transmitted_areas=broadcast_message.areas,
-            # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it
-            # include a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
-            transmitted_sender='notifications.service.gov.uk',
-
-            # TODO: Should this be set to now? Or the original starts_at?
-            transmitted_starts_at=broadcast_message.starts_at,
-            transmitted_finishes_at=broadcast_message.finishes_at,
-        )
-
-        dao_save_object(event)
-
-        send_broadcast_event.apply_async(
-            kwargs={'broadcast_event_id': str(event.id)},
-            queue=QueueNames.BROADCASTS
-        )
-    elif broadcast_message.stubbed != service.restricted:
-        # It's possible for a service to create a broadcast in trial mode, and then approve it after the
-        # service is live (or vice versa). We don't think it's safe to send such broadcasts, as the service
-        # has changed since they were created. Log an error instead.
-        current_app.logger.error(
-            f'Broadcast event not created. Stubbed status of broadcast message was {broadcast_message.stubbed}'
-            f' but service was {"in trial mode" if service.restricted else "live"}'
-        )
--- a/app/celery/broadcast_message_tasks.py
+++ b/app/celery/broadcast_message_tasks.py
@@ -1,230 +1,45 @@
-from datetime import datetime
-
+import requests
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd

 from app import cbc_proxy_client, notify_celery
-from app.clients.cbc_proxy import CBCProxyRetryableException
-from app.config import QueueNames, TaskNames
-from app.dao.broadcast_message_dao import (
-    create_broadcast_provider_message,
-    dao_get_broadcast_event_by_id,
-    update_broadcast_provider_message_status,
-)
-from app.models import (
-    BroadcastEventMessageType,
-    BroadcastProvider,
-    BroadcastProviderMessageStatus,
-)
-from app.utils import format_sequential_number

-
-class BroadcastIntegrityError(Exception):
-    pass
-
-
-def get_retry_delay(retry_count):
-    """
-    Given a count of retries so far, return a delay for the next one.
-    `retry_count` should be 0 the first time a task fails.
-    """
-    # TODO: replace with celery's built in exponential backoff
-
-    # 2 to the power of x. 1, 2, 4, 8, 16, 32, ...
-    delay = 2**retry_count
-    # never wait longer than 4 minutes
-    return min(delay, 240)
-
-
-def check_event_is_authorised_to_be_sent(broadcast_event, provider):
-    if not broadcast_event.service.active:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is suspended'
-        )
-
-    if broadcast_event.service.restricted:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is not live'
-        )
-
-    if broadcast_event.broadcast_message.stubbed:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the broadcast message is stubbed'
-        )
-
-
-def check_event_makes_sense_in_sequence(broadcast_event, provider):
-    """
-    If any previous event hasn't sent yet for that provider, then we shouldn't send the current event. Instead, fail and
-    raise a zendesk ticket - so that a notify team member can assess the state of the previous messages, and if
-    necessary, can replay the `send_broadcast_provider_message` task if the previous message has now been sent.
-
-    Note: This is called before the new broadcast_provider_message is created.
-
-    # Help, I've come across this code following a pagerduty alert, what should I do?
-
-    1. Find the failing broadcast_provider_message associated with the previous event that caused this to trip.
-    2. If that provider message is still failing to send, fix the issue causing that. The task to send that previous
-       message might still be retrying in the background - look for logs related to that task.
-    3. If that provider message has sent succesfully, you might need to send this task off depending on context. This
-       might not always be true though, for example, it may not be necessary to send a cancel if the original alert has
-       already expired.
-    4. If you need to re-send this task off again, you'll need to run the following command on paas:
-       `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)`
-    """
-    current_provider_message = broadcast_event.get_provider_message(provider)
-    # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'It is in status {current_provider_message.status}'
-        )
-
-    if broadcast_event.transmitted_finishes_at < datetime.utcnow():
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'The expiry time of {broadcast_event.transmitted_finishes_at} has already passed'
-        )
-
-    # get events sorted from earliest to latest
-    events = sorted(broadcast_event.broadcast_message.events, key=lambda x: x.sent_at)
-
-    for prev_event in events:
-        if prev_event.id != broadcast_event.id and prev_event.sent_at < broadcast_event.sent_at:
-            # get the record from when that event was sent to the same provider
-            prev_provider_message = prev_event.get_provider_message(provider)
-
-            # the previous message hasn't even got round to running `send_broadcast_provider_message` yet.
-            if not prev_provider_message:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has no provider_message for provider {provider} yet.\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-            # if there's a previous message that has started but not finished sending (whether it fatally errored or is
-            # currently retrying)
-            if prev_provider_message.status != BroadcastProviderMessageStatus.ACK:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has not finished sending to provider {provider} yet.\n' +
-                    f'It is currently in status "{prev_provider_message.status}".\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
+from app.models import BroadcastEventMessageType
+from app.dao.broadcast_message_dao import dao_get_broadcast_event_by_id


@notify_celery.task(name="send-broadcast-event")
-def send_broadcast_event(broadcast_event_id):
+@statsd(namespace="tasks")
+def send_broadcast_event(broadcast_event_id, provider='stub-1'):
    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)

-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
-
-    for provider in broadcast_event.service.get_available_broadcast_providers():
-        send_broadcast_provider_message.apply_async(
-            kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider},
-            queue=QueueNames.BROADCASTS
-        )
-
-
-# max_retries=None: retry forever
-@notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None)
-def send_broadcast_provider_message(self, broadcast_event_id, provider):
-    if not current_app.config['CBC_PROXY_ENABLED']:
+    if broadcast_event.message_type == BroadcastEventMessageType.ALERT:
        current_app.logger.info(
-            "CBC Proxy disabled, not sending broadcast_provider_message for "
-            f"broadcast_event_id {broadcast_event_id} with provider {provider}"
+            f'invoking cbc proxy to send '
+            f'broadcast_event {broadcast_event.reference} '
+            f'msgType {broadcast_event.message_type} to {provider}'
        )
-        return

-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    check_event_is_authorised_to_be_sent(broadcast_event, provider)
-    check_event_makes_sense_in_sequence(broadcast_event, provider)
-
-    # the broadcast_provider_message may already exist if we retried previously
-    broadcast_provider_message = broadcast_event.get_provider_message(provider)
-    if broadcast_provider_message is None:
-        broadcast_provider_message = create_broadcast_provider_message(broadcast_event, provider)
-
-    formatted_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        formatted_message_number = format_sequential_number(broadcast_provider_message.message_number)
+        cbc_proxy_client.create_and_send_broadcast(
+            identifier=str(broadcast_event.id),
+            headline="GOV.UK Notify Broadcast",
+            description=broadcast_event.transmitted_content['body'],
+        )

    current_app.logger.info(
-        f'Invoking cbc proxy to send broadcast_provider_message with ID of {broadcast_provider_message.id} '
-        f'and broadcast_event ID of {broadcast_event_id} '
-        f'msgType {broadcast_event.message_type}'
+        f'sending broadcast_event {broadcast_event.reference} '
+        f'msgType {broadcast_event.message_type} to {provider}'
    )

-    areas = [
-        {"polygon": polygon}
-        for polygon in broadcast_event.transmitted_areas["simple_polygons"]
-    ]
+    payload = broadcast_event.serialize()

-    cbc_proxy_provider_client = cbc_proxy_client.get_proxy(provider)
+    resp = requests.post(
+        f'{current_app.config["CBC_PROXY_URL"]}/broadcasts/events/{provider}',
+        json=payload
+    )
+    resp.raise_for_status()

-    try:
-        if broadcast_event.message_type == BroadcastEventMessageType.ALERT:
-            cbc_proxy_provider_client.create_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.UPDATE:
-            cbc_proxy_provider_client.update_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                # We think an alert update should always go out on the same channel that created the alert
-                # We recognise there is a small risk with this code here that if the services channel was
-                # changed between an alert being sent out and then updated, then something might go wrong
-                # but we are relying on service channels changing almost never, and not mid incident
-                # We may consider in the future, changing this such that we store the channel a broadcast was
-                # sent on on the broadcast message itself and pick the value from there instead of the service
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.CANCEL:
-            cbc_proxy_provider_client.cancel_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-            )
-    except CBCProxyRetryableException as exc:
-        delay = get_retry_delay(self.request.retries)
-        current_app.logger.exception(
-            f'Retrying send_broadcast_provider_message for broadcast event {broadcast_event_id}, '
-            f'provider message {broadcast_provider_message.id}, provider {provider} in {delay} seconds'
-        )
-
-        self.retry(
-            exc=exc,
-            countdown=delay,
-            queue=QueueNames.BROADCASTS,
-        )
-
-    update_broadcast_provider_message_status(broadcast_provider_message, status=BroadcastProviderMessageStatus.ACK)
-
-
-@notify_celery.task(name='trigger-link-test')
-def trigger_link_test(provider):
-    cbc_proxy_client.get_proxy(provider).send_link_test()
+    current_app.logger.info(
+        f'broadcast_event {broadcast_event.reference} '
+        f'msgType {broadcast_event.message_type} sent to {provider}'
+    )
--- a/app/celery/celery.py
+++ b/app/celery/celery.py
@@ -0,0 +1,78 @@
+import time
+
+from gds_metrics.metrics import Histogram
+from celery import Celery, Task
+from celery.signals import worker_process_shutdown
+from flask import g, request
+from flask.ctx import has_request_context
+
+
+@worker_process_shutdown.connect
+def log_on_worker_shutdown(sender, signal, pid, exitcode, **kwargs):
+    # imported here to avoid circular imports
+    from app import notify_celery
+
+    # if the worker has already restarted at least once, then we no longer have app context and current_app won't work
+    # to create a new one. Instead we have to create a new app context from the original flask app and use that instead.
+    with notify_celery._app.app_context():
+        # if the worker has restarted
+        notify_celery._app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
+
+
+def make_task(app):
+    SQS_APPLY_ASYNC_DURATION_SECONDS = Histogram(
+        'sqs_apply_async_duration_seconds',
+        'Time taken to put task on queue',
+        ['task_name']
+    )
+
+    class NotifyTask(Task):
+        abstract = True
+        start = None
+
+        def on_success(self, retval, task_id, args, kwargs):
+            elapsed_time = time.time() - self.start
+            app.logger.info(
+                "{task_name} took {time}".format(
+                    task_name=self.name, time="{0:.4f}".format(elapsed_time)
+                )
+            )
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            # ensure task will log exceptions to correct handlers
+            app.logger.exception('Celery task: {} failed'.format(self.name))
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def __call__(self, *args, **kwargs):
+            # ensure task has flask context to access config, logger, etc
+            with app.app_context():
+                self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
+                return super().__call__(*args, **kwargs)
+
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            with SQS_APPLY_ASYNC_DURATION_SECONDS.labels(self.name).time():
+                return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
+    return NotifyTask
+
+
+class NotifyCelery(Celery):
+
+    def init_app(self, app):
+        super().__init__(
+            app.import_name,
+            broker=app.config['BROKER_URL'],
+            task_cls=make_task(app),
+        )
+
+        self.conf.update(app.config)
+        self._app = app
--- a/app/celery/letters_pdf_tasks.py
+++ b/app/celery/letters_pdf_tasks.py
@@ -1,47 +1,41 @@
-from base64 import urlsafe_b64encode
 from datetime import datetime, timedelta
 from hashlib import sha512
+from base64 import urlsafe_b64encode

 from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
-from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.postal_address import PostalAddress
+
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.timezones import convert_utc_to_bst

 from app import encryption, notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
-from app.cronitor import cronitor
 from app.dao.notifications_dao import (
-    dao_get_letters_and_sheets_volume_by_postage,
-    dao_get_letters_to_be_printed,
-    dao_get_notification_by_reference,
-    dao_update_notification,
-    dao_update_notifications_by_reference,
    get_notification_by_id,
    update_notification_status_by_id,
+    dao_update_notification,
+    dao_get_notification_by_reference,
+    dao_update_notifications_by_reference,
+    dao_get_letters_to_be_printed,
 )
-from app.dao.templates_dao import dao_get_template_by_id
+from app.letters.utils import get_letter_pdf_filename
 from app.errors import VirusScanError
 from app.exceptions import NotificationTechnicalFailureException
 from app.letters.utils import (
-    LetterPDFNotFound,
-    ScanErrorType,
-    find_letter_pdf_in_s3,
-    generate_letter_pdf_filename,
    get_billable_units_for_letter_page_count,
-    get_file_names_from_error_bucket,
-    get_folder_name,
    get_reference_from_filename,
-    move_error_pdf_to_scan_bucket,
+    ScanErrorType,
    move_failed_pdf,
    move_sanitised_letter_to_test_or_live_pdf_bucket,
    move_scan_to_invalid_pdf_bucket,
+    move_error_pdf_to_scan_bucket,
+    get_file_names_from_error_bucket,
 )
 from app.models import (
    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_POSTAGE_TYPES,
-    KEY_TYPE_NORMAL,
    KEY_TYPE_TEST,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
@@ -51,16 +45,24 @@ from app.models import (
    NOTIFICATION_VIRUS_SCAN_FAILED,
    POSTAGE_TYPES,
    RESOLVE_POSTAGE_FOR_FILE_NAME,
-    Service,
-)
+    INTERNATIONAL_POSTAGE_TYPES)
+from app.cronitor import cronitor
+
+
+@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def create_letters_pdf(self, notification_id):
+    get_pdf_for_templated_letter(notification_id)


@notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def get_pdf_for_templated_letter(self, notification_id):
    try:
        notification = get_notification_by_id(notification_id, _raise=True)
-        letter_filename = generate_letter_pdf_filename(
+        letter_filename = get_letter_pdf_filename(
            reference=notification.reference,
+            crown=notification.service.crown,
            created_at=notification.created_at,
            ignore_folder=notification.key_type == KEY_TYPE_TEST,
            postage=notification.postage
@@ -86,12 +88,12 @@ def get_pdf_for_templated_letter(self, notification_id):
            args=(encrypted_data,),
            queue=QueueNames.SANITISE_LETTERS
        )
-    except Exception as e:
+    except Exception:
        try:
            current_app.logger.exception(
                f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
            )
-            self.retry(exc=e, queue=QueueNames.RETRY)
+            self.retry(queue=QueueNames.RETRY)
        except self.MaxRetriesExceededError:
            message = f"RETRY FAILED: Max retries reached. " \
                      f"The task create-letter-pdf failed for notification id {notification_id}. " \
@@ -101,6 +103,7 @@ def get_pdf_for_templated_letter(self, notification_id):


@notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def update_billable_units_for_letter(self, notification_id, page_count):
    notification = get_notification_by_id(notification_id, _raise=True)

@@ -116,16 +119,6 @@ def update_billable_units_for_letter(self, notification_id, page_count):
        )


-@notify_celery.task(
-    bind=True, name="update-validation-failed-for-templated-letter", max_retries=15, default_retry_delay=300
-)
-def update_validation_failed_for_templated_letter(self, notification_id, page_count):
-    notification = get_notification_by_id(notification_id, _raise=True)
-    notification.status = NOTIFICATION_VALIDATION_FAILED
-    dao_update_notification(notification)
-    current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}")
-
-
@notify_celery.task(name='collate-letter-pdfs-to-be-sent')
@cronitor("collate-letter-pdfs-to-be-sent")
 def collate_letter_pdfs_to_be_sent():
@@ -136,6 +129,7 @@ def collate_letter_pdfs_to_be_sent():
    that have not yet been sent.
    If run after midnight, it will collect up letters created before 5:30pm the day before.
    """
+    current_app.logger.info("starting collate-letter-pdfs-to-be-sent")
    print_run_date = convert_utc_to_bst(datetime.utcnow())
    if print_run_date.time() < LETTER_PROCESSING_DEADLINE:
        print_run_date = print_run_date - timedelta(days=1)
@@ -143,27 +137,22 @@ def collate_letter_pdfs_to_be_sent():
    print_run_deadline = print_run_date.replace(
        hour=17, minute=30, second=0, microsecond=0
    )
-    _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)
-
    for postage in POSTAGE_TYPES:
        current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)

        for i, letters in enumerate(group_letters(letters_to_print)):
            filenames = [letter['Key'] for letter in letters]
-
            service_id = letters[0]['ServiceId']
-            organisation_id = letters[0]['OrganisationId']

            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format(
+            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.ZIP'.format(
                date=print_run_deadline.strftime("%Y-%m-%d"),
                postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
                num=i + 1,
                hash=hash,
-                service_id=service_id,
-                organisation_id=organisation_id
+                service_id=service_id
            )

            current_app.logger.info(
@@ -187,71 +176,23 @@ def collate_letter_pdfs_to_be_sent():
    current_app.logger.info("finished collate-letter-pdfs-to-be-sent")


-def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
-    letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
-    send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date())
-
-
-def send_letters_volume_email_to_dvla(letters_volumes, date):
-    personalisation = {
-        'total_volume': 0,
-        'first_class_volume': 0,
-        'second_class_volume': 0,
-        'international_volume': 0,
-        'total_sheets': 0,
-        'first_class_sheets': 0,
-        "second_class_sheets": 0,
-        'international_sheets': 0,
-        'date': date.strftime("%d %B %Y")
-    }
-    for item in letters_volumes:
-        personalisation['total_volume'] += item.letters_count
-        personalisation['total_sheets'] += item.sheets_count
-        if f"{item.postage}_class_volume" in personalisation:
-            personalisation[f"{item.postage}_class_volume"] = item.letters_count
-            personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
-        else:
-            personalisation["international_volume"] += item.letters_count
-            personalisation["international_sheets"] += item.sheets_count
-
-    template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
-    recipients = current_app.config['DVLA_EMAIL_ADDRESSES']
-    reply_to = template.service.get_default_reply_to_email_address()
-    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
-
-    # avoid circular imports:
-    from app.notifications.process_notifications import (
-        persist_notification,
-        send_notification_to_queue,
-    )
-    for recipient in recipients:
-        saved_notification = persist_notification(
-            template_id=template.id,
-            template_version=template.version,
-            recipient=recipient,
-            service=service,
-            personalisation=personalisation,
-            notification_type=template.template_type,
-            api_key_id=None,
-            key_type=KEY_TYPE_NORMAL,
-            reply_to_text=reply_to
-        )
-
-        send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
-
-
 def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
    for letter in letters_awaiting_sending:
        try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
+            letter_file_name = get_letter_pdf_filename(
+                reference=letter.reference,
+                crown=letter.service.crown,
+                created_at=letter.created_at,
+                postage=letter.postage
+            )
+            letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
            yield {
-                "Key": letter_pdf.key,
-                "Size": letter_pdf.size,
-                "ServiceId": str(letter.service_id),
-                "OrganisationId": str(letter.service.organisation_id)
+                "Key": letter_file_name,
+                "Size": letter_head['ContentLength'],
+                "ServiceId": str(letter.service.id)
            }
-        except (BotoClientError, LetterPDFNotFound) as e:
+        except BotoClientError as e:
            current_app.logger.exception(
                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)

@@ -377,8 +318,9 @@ def process_sanitised_letter(self, sanitise_data):

        # The original filename could be wrong because we didn't know the postage.
        # Now we know if the letter is international, we can check what the filename should be.
-        upload_file_name = generate_letter_pdf_filename(
+        upload_file_name = get_letter_pdf_filename(
            reference=notification.reference,
+            crown=notification.service.crown,
            created_at=notification.created_at,
            ignore_folder=True,
            postage=notification.postage
@@ -489,7 +431,6 @@ def update_letter_pdf_status(reference, status, billable_units, recipient_addres
        update_dict.update({'postage': postage, 'international': True})
    if recipient_address:
        update_dict['to'] = recipient_address
-        update_dict['normalised_to'] = ''.join(recipient_address.split()).lower()
    return dao_update_notifications_by_reference(
        references=[reference],
        update_dict=update_dict)[0]
@@ -534,37 +475,3 @@ def replay_letters_in_error(filename=None):
                    [filename],
                    queue=QueueNames.LETTERS
                )
-
-
-@notify_celery.task(name='resanitise-pdf')
-def resanitise_pdf(notification_id):
-    """
-    `notification_id` is the notification id for a PDF letter which was either uploaded or sent using the API.
-
-    This task calls the `recreate_pdf_for_precompiled_letter` template preview task which recreates the
-    PDF for a letter which is already sanitised and in the letters-pdf bucket. The new file that is generated
-    will then overwrite the existing letter in the letters-pdf bucket.
-    """
-    notification = get_notification_by_id(notification_id)
-
-    # folder_name is the folder that the letter is in the letters-pdf bucket e.g. '2021-10-10/'
-    folder_name = get_folder_name(notification.created_at)
-
-    filename = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-    notify_celery.send_task(
-        name=TaskNames.RECREATE_PDF_FOR_PRECOMPILED_LETTER,
-        kwargs={
-            'notification_id': str(notification.id),
-            'file_location': f'{folder_name}{filename}',
-            'allow_international_letters': notification.service.has_permission(
-                INTERNATIONAL_LETTERS
-            ),
-        },
-        queue=QueueNames.SANITISE_LETTERS,
-    )
--- a/app/celery/nightly_tasks.py
+++ b/app/celery/nightly_tasks.py
@@ -1,54 +1,55 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)

 import pytz
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from notifications_utils.timezones import convert_utc_to_bst
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError

-from app import notify_celery, statsd_client, zendesk_client
+from app import notify_celery, performance_platform_client, zendesk_client
 from app.aws import s3
-from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
+from app.celery.service_callback_tasks import (
+    send_delivery_status_to_service,
+    create_delivery_status_callback_data,
+)
 from app.config import QueueNames
-from app.cronitor import cronitor
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
-    dao_archive_job,
    dao_get_jobs_older_than_data_retention,
+    dao_archive_job
 )
 from app.dao.notifications_dao import (
-    dao_get_notifications_processing_time_stats,
    dao_timeout_notifications,
-    get_service_ids_with_notifications_before,
-    move_notifications_to_notification_history,
-)
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_for_all_services_by_notification_type,
+    delete_notifications_older_than_retention_by_type,
 )
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_SENDING,
-    SMS_TYPE,
-    FactProcessingTime,
    Notification,
+    NOTIFICATION_SENDING,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    KEY_TYPE_NORMAL
 )
+from app.performance_platform import total_sent_notifications, processing_time
+from app.cronitor import cronitor
 from app.utils import get_london_midnight_in_utc


@notify_celery.task(name="remove_sms_email_jobs")
@cronitor("remove_sms_email_jobs")
+@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
    _remove_csv_files([EMAIL_TYPE, SMS_TYPE])


@notify_celery.task(name="remove_letter_jobs")
@cronitor("remove_letter_jobs")
+@statsd(namespace="tasks")
 def remove_letter_csv_files():
    _remove_csv_files([LETTER_TYPE])

@@ -62,111 +63,149 @@ def _remove_csv_files(job_types):


@notify_celery.task(name="delete-notifications-older-than-retention")
+@statsd(namespace="tasks")
 def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_email_notifications_older_than_retention()
+    delete_sms_notifications_older_than_retention()
+    delete_letter_notifications_older_than_retention()


@notify_celery.task(name="delete-sms-notifications")
@cronitor("delete-sms-notifications")
+@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('sms')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('sms')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise


@notify_celery.task(name="delete-email-notifications")
@cronitor("delete-email-notifications")
+@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('email')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('email')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete email notifications")
+        raise


@notify_celery.task(name="delete-letter-notifications")
@cronitor("delete-letter-notifications")
+@statsd(namespace="tasks")
 def delete_letter_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('letter')
-
-
-def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
-
-    for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_london_midnight_in_utc(
-            convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
-        )
-
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': f.service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': day_to_delete_backwards_from
-        })
-
-    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7))
-    service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
-
-    # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
-    # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
-        notification_type,
-        seven_days_ago
-    )
-
-    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
-
-    for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': seven_days_ago
-        })
-
-    current_app.logger.info(
-        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
-        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
-        f'{len(service_ids_to_purge)} services without flexible data retention'
-    )
-
-
-@notify_celery.task(name='delete-notifications-for-service-and-type')
-def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
-    start = datetime.utcnow()
-    num_deleted = move_notifications_to_notification_history(
-        notification_type,
-        service_id,
-        datetime_to_delete_before,
-    )
-    if num_deleted:
-        end = datetime.utcnow()
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('letter')
        current_app.logger.info(
-            f'delete-notifications-for-service-and-type: '
-            f'service: {service_id}, '
-            f'notification_type: {notification_type}, '
-            f'count deleted: {num_deleted}, '
-            f'duration: {(end - start).seconds} seconds'
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
+                'letter',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete letter notifications")
+        raise


@notify_celery.task(name='timeout-sending-notifications')
@cronitor('timeout-sending-notifications')
+@statsd(namespace="tasks")
 def timeout_notifications():
-    notifications = ['dummy value so len() > 0']
+    technical_failure_notifications, temporary_failure_notifications = \
+        dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))

-    cutoff_time = datetime.utcnow() - timedelta(
-        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
+    notifications = technical_failure_notifications + temporary_failure_notifications
+    for notification in notifications:
+        # queue callback task only if the service_callback_api exists
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
+
+    current_app.logger.info(
+        "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    if technical_failure_notifications:
+        message = "{} notifications have been updated to technical-failure because they " \
+                  "have timed out and are still in created.Notification ids: {}".format(
+                      len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
+        raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@cronitor('send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_platform_stats(date=None):
+    # date is a string in the format of "YYYY-MM-DD"
+    if date is None:
+        date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        date = datetime.strptime(date, "%Y-%m-%d").date()
+
+    if performance_platform_client.active:
+
+        send_total_sent_notifications_to_performance_platform(bst_date=date)
+        processing_time.send_processing_time_to_performance_platform(bst_date=date)
+
+
+def send_total_sent_notifications_to_performance_platform(bst_date):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
+    start_time = get_london_midnight_in_utc(bst_date)
+
+    email_sent_count = count_dict['email']
+    sms_sent_count = count_dict['sms']
+    letter_sent_count = count_dict['letter']
+
+    current_app.logger.info(
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
    )

-    while len(notifications) > 0:
-        notifications = dao_timeout_notifications(cutoff_time)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'sms',
+        sms_sent_count
+    )

-        for notification in notifications:
-            statsd_client.incr(f'timeout-sending.{notification.sent_by}')
-            check_and_queue_callback_task(notification)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'email',
+        email_sent_count
+    )

-        current_app.logger.info(
-            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'letter',
+        letter_sent_count
+    )


@notify_celery.task(name="delete-inbound-sms")
@cronitor("delete-inbound-sms")
+@statsd(namespace="tasks")
 def delete_inbound_sms():
    try:
        start = datetime.utcnow()
@@ -185,6 +224,7 @@ def delete_inbound_sms():

@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
@cronitor("raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
    still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()

@@ -196,16 +236,11 @@ def raise_alert_if_letter_notifications_still_sending():
        # Only send alerts in production
        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
            message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"  # noqa
-
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
-                email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
        else:
            current_app.logger.info(message)

@@ -231,11 +266,17 @@ def get_letter_notifications_still_sending_when_they_shouldnt_be():
        func.date(Notification.sent_at) <= expected_sent_date
    )

+    if today.isoweekday() in {2, 4}:  # on tue, thu, we only care about first class letters
+        q = q.filter(
+            Notification.postage == 'first'
+        )
+
    return q.count(), expected_sent_date


@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
@cronitor('raise-alert-if-no-letter-ack-file')
+@statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
    # get a list of zip files since yesterday
    zip_file_set = set()
@@ -253,57 +294,33 @@ def letter_raise_alert_if_no_ack_file_for_zip():

    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                              subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
-
-    message = '\n'.join([
-        "Letter ack file does not contain all zip files sent."
-        "",
-        f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n",  # noqa
-        f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent",  # noqa
-        f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
-        "",
-        f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
-    ])
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))

+    message = (
+        "Letter ack file does not contain all zip files sent. "
+        "Missing ack for zip files: {}, "
+        "pdf bucket: {}, subfolder: {}, "
+        "ack bucket: {}"
+    ).format(
+        str(sorted(zip_file_set - ack_file_set)),
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
+        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
+    )
    # strip empty element before comparison
    ack_file_set.discard('')
    zip_file_set.discard('')

    if len(zip_file_set - ack_file_set) > 0:
        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
                subject="Letter acknowledge error",
                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
        current_app.logger.error(message)

    if len(ack_file_set - zip_file_set) > 0:
        current_app.logger.info(
            "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
        )
-
-
-@notify_celery.task(name='save-daily-notification-processing-time')
-@cronitor("save-daily-notification-processing-time")
-def save_daily_notification_processing_time(bst_date=None):
-    # bst_date is a string in the format of "YYYY-MM-DD"
-    if bst_date is None:
-        # if a date is not provided, we run against yesterdays data
-        bst_date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        bst_date = datetime.strptime(bst_date, "%Y-%m-%d").date()
-
-    start_time = get_london_midnight_in_utc(bst_date)
-    end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
-    result = dao_get_notifications_processing_time_stats(start_time, end_time)
-    insert_update_processing_time(
-        FactProcessingTime(
-            bst_date=bst_date,
-            messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs
-        )
-    )
--- a/app/celery/process_ses_receipts_tasks.py
+++ b/app/celery/process_ses_receipts_tasks.py
@@ -3,95 +3,75 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound

 from app import notify_celery, statsd_client
-from app.celery.service_callback_tasks import (
-    create_complaint_callback_data,
-    create_delivery_status_callback_data,
-    send_complaint_to_service,
-    send_delivery_status_to_service,
-)
 from app.config import QueueNames
+from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.dao.complaint_dao import save_complaint
-from app.dao.notifications_dao import dao_get_notification_history_by_reference
-from app.dao.service_callback_api_dao import (
-    get_service_complaint_callback_api_for_service,
-    get_service_delivery_status_callback_api_for_service,
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
+
+from app.notifications.notifications_ses_callback import (
+    determine_notification_bounce_type,
+    handle_complaint,
+    _check_and_queue_complaint_callback_task,
+    _check_and_queue_callback_task,
 )
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint


@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_ses_results(self, response):
    try:
-        ses_message = json.loads(response["Message"])
-        notification_type = ses_message["notificationType"]
-        # TODO remove after smoke testing on prod is implemented
-        current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}")
-        bounce_message = None
+        ses_message = json.loads(response['Message'])
+        notification_type = ses_message['notificationType']

        if notification_type == 'Bounce':
-            bounce_message = determine_notification_bounce_type(ses_message)
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
        elif notification_type == 'Complaint':
            _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
            return True

-        aws_response_dict = get_aws_responses(ses_message)
+        aws_response_dict = get_aws_responses(notification_type)

-        notification_status = aws_response_dict["notification_status"]
-        reference = ses_message["mail"]["messageId"]
+        notification_status = aws_response_dict['notification_status']
+        reference = ses_message['mail']['messageId']

        try:
-            notification = notifications_dao.dao_get_notification_by_reference(reference)
+            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
        except NoResultFound:
-            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
+            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
            if datetime.utcnow() - message_time < timedelta(minutes=5):
                current_app.logger.info(
-                    f"notification not found for reference: {reference} (while attempting update to {notification_status}). "
+                    f"notification not found for reference: {reference} (update to {notification_status}). "
                    f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
                )
                self.retry(queue=QueueNames.RETRY)
            else:
                current_app.logger.warning(
-                    "notification not found for reference: {} (while attempting update to {})".format(reference, notification_status)
+                    f"notification not found for reference: {reference} (update to {notification_status})"
                )
            return

-        if bounce_message:
-            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
-
-        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
+        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
            notifications_dao._duplicate_update_warning(
-                notification, 
-                notification_status
+                notification=notification,
+                status=notification_status
            )
            return
-
-        notifications_dao._update_notification_status(
-            notification=notification,
-            status=notification_status,
-            provider_response=aws_response_dict["provider_response"],
-        )
-
-        if not aws_response_dict["success"]:
-            current_app.logger.info(
-                "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
-                    notification.id, reference, aws_response_dict["message"]
-                )
-            )
        else:
-            current_app.logger.info(
-                "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
+            notifications_dao.dao_update_notifications_by_reference(
+                references=[reference],
+                update_dict={'status': notification_status}
            )

-        statsd_client.incr("callback.ses.{}".format(notification_status))
+        statsd_client.incr('callback.ses.{}'.format(notification_status))

        if notification.sent_at:
-            statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at)
+            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)

-        check_and_queue_callback_task(notification)
+        _check_and_queue_callback_task(notification)

        return True

@@ -99,133 +79,5 @@ def process_ses_results(self, response):
        raise

    except Exception as e:
-        current_app.logger.exception("Error processing SES results: {}".format(type(e)))
+        current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
        self.retry(queue=QueueNames.RETRY)
-
-def determine_notification_bounce_type(ses_message):
-    notification_type = ses_message["notificationType"]
-    if notification_type in ["Delivery", "Complaint"]:
-        return notification_type
-
-    if notification_type != "Bounce":
-        raise KeyError(f"Unhandled sns notification type {notification_type}")
-
-    remove_emails_from_bounce(ses_message)
-    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
-    if ses_message["bounce"]["bounceType"] == "Permanent":
-        return "Permanent"
-    return "Temporary"
-
-def determine_notification_type(ses_message):
-    notification_type = ses_message["notificationType"]
-    if notification_type not in ["Bounce","Complaint","Delivery"]:
-        raise KeyError(f"Unhandled sns notification type {notification_type}")
-    if notification_type == 'Bounce':
-        return determine_notification_bounce_type(ses_message)
-    return notification_type
-
-def _determine_provider_response(ses_message):
-    if ses_message["notificationType"] != "Bounce":
-        return None
-
-    bounce_type = ses_message["bounce"]["bounceType"]
-    bounce_subtype = ses_message["bounce"]["bounceSubType"]
-
-    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
-    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
-        return "The email address is on our email provider suppression list"
-    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
-        return "The email address is on the GC Notify suppression list"
-    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
-        return "The email was rejected because of its attachments"
-
-    return None
-
-
-def get_aws_responses(ses_message):
-    status = determine_notification_type(ses_message)
-
-    base = {
-        "Permanent": {
-            "message": "Hard bounced",
-            "success": False,
-            "notification_status": "permanent-failure",
-        },
-        "Temporary": {
-            "message": "Soft bounced",
-            "success": False,
-            "notification_status": "temporary-failure",
-        },
-        "Delivery": {
-            "message": "Delivered",
-            "success": True,
-            "notification_status": "delivered",
-        },
-        "Complaint": {
-            "message": "Complaint",
-            "success": True,
-            "notification_status": "delivered",
-        },
-    }[status]
-
-    base["provider_response"] = _determine_provider_response(ses_message)
-
-    return base
-
-
-def handle_complaint(ses_message):
-    recipient_email = remove_emails_from_complaint(ses_message)[0]
-    current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
-    try:
-        reference = ses_message["mail"]["messageId"]
-    except KeyError as e:
-        current_app.logger.exception(f"Complaint from SES failed to get reference from message with error: {e}")
-        return
-    notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get("complaint", None)
-
-    complaint = Complaint(
-        notification_id=notification.id,
-        service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
-        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
-        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
-    )
-    save_complaint(complaint)
-    return complaint, notification, recipient_email
-
-
-def remove_mail_headers(dict_to_edit):
-    if dict_to_edit["mail"].get("headers"):
-        dict_to_edit["mail"].pop("headers")
-    if dict_to_edit["mail"].get("commonHeaders"):
-        dict_to_edit["mail"].pop("commonHeaders")
-
-
-def remove_emails_from_bounce(bounce_dict):
-    remove_mail_headers(bounce_dict)
-    bounce_dict["mail"].pop("destination", None)
-    bounce_dict["bounce"].pop("bouncedRecipients", None)
-
-
-def remove_emails_from_complaint(complaint_dict):
-    remove_mail_headers(complaint_dict)
-    complaint_dict["complaint"].pop("complainedRecipients")
-    return complaint_dict["mail"].pop("destination")
-
-
-def check_and_queue_callback_task(notification):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
-
-
-def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
-        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
-        
--- a/app/celery/process_sms_client_response_tasks.py
+++ b/app/celery/process_sms_client_response_tasks.py
@@ -1,25 +1,29 @@
 import uuid
 from datetime import datetime

-import pytest
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate

 from app import notify_celery, statsd_client
-from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.clients import ClientException
+from app.clients.sms.firetext import get_firetext_responses
+from app.clients.sms.mmg import get_mmg_responses
+from app.celery.service_callback_tasks import send_delivery_status_to_service, create_delivery_status_callback_data
+from app.config import QueueNames
 from app.dao import notifications_dao
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING

 sms_response_mapper = {
-    # 'MMG': get_mmg_responses,
-    # 'Firetext': get_firetext_responses,
+    'MMG': get_mmg_responses,
+    'Firetext': get_firetext_responses
 }


-# gUpdate with new providers")
@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
    # validate reference
    try:
@@ -68,7 +72,7 @@ def _process_for_status(notification_status, client_name, provider_reference, de

    if notification.sent_at:
        statsd_client.timing_with_dates(
-            f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
+            'callback.{}.elapsed-time'.format(client_name.lower()),
            datetime.utcnow(),
            notification.sent_at
        )
@@ -87,4 +91,9 @@ def _process_for_status(notification_status, client_name, provider_reference, de
        notifications_dao.dao_update_notification(notification)

    if notification_status != NOTIFICATION_PENDING:
-        check_and_queue_callback_task(notification)
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        # queue callback task only if the service_callback_api exists
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -1,11 +1,11 @@
 from flask import current_app
+from notifications_utils.recipients import InvalidEmailError
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound

 from app import notify_celery
-from app.clients.email import EmailClientNonRetryableException
-from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
-from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
+from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.delivery import send_to_providers
@@ -14,6 +14,7 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE


@notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
    try:
        current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -21,18 +22,11 @@ def deliver_sms(self, notification_id):
        if not notification:
            raise NoResultFound()
        send_to_providers.send_sms_to_provider(notification)
-    except Exception as e:
-        if isinstance(e, SmsClientResponseException):
-            current_app.logger.warning(
-                "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True
-            )
-        else:
+    except Exception:
+        try:
            current_app.logger.exception(
                "SMS notification delivery for id: {} failed".format(notification_id)
            )
-
-        try:
            if self.request.retries == 0:
                self.retry(queue=QueueNames.RETRY, countdown=0)
            else:
@@ -45,6 +39,7 @@ def deliver_sms(self, notification_id):


@notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
    try:
        current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
@@ -52,10 +47,8 @@ def deliver_email(self, notification_id):
        if not notification:
            raise NoResultFound()
        send_to_providers.send_email_to_provider(notification)
-    except EmailClientNonRetryableException as e:
-        current_app.logger.exception(
-            f"Email notification {notification_id} failed: {e}"
-        )
+    except InvalidEmailError as e:
+        current_app.logger.exception(e)
        update_notification_status_by_id(notification_id, 'technical-failure')
    except Exception as e:
        try:
--- a/app/celery/reporting_tasks.py
+++ b/app/celery/reporting_tasks.py
@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta

 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst

 from app import notify_celery
@@ -8,16 +9,21 @@ from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_billing_dao import (
    fetch_billing_data_for_day,
-    update_fact_billing,
+    update_fact_billing
+)
+from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    LETTER_TYPE,
 )
-from app.dao.fact_notification_status_dao import update_fact_notification_status
-from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE


@notify_celery.task(name="create-nightly-billing")
@cronitor("create-nightly-billing")
+@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
+    current_app.logger.info("create-nightly-billing task: started")
    # day_start is a datetime.date() object. e.g.
    # up to 4 days of data counting back from day_start is consolidated
    if day_start is None:
@@ -25,7 +31,7 @@ def create_nightly_billing(day_start=None):
    else:
        # When calling the task its a string in the format of "YYYY-MM-DD"
        day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 10):
+    for i in range(0, 4):
        process_day = (day_start - timedelta(days=i)).isoformat()

        create_nightly_billing_for_day.apply_async(
@@ -38,6 +44,7 @@ def create_nightly_billing(day_start=None):


@notify_celery.task(name="create-nightly-billing-for-day")
+@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
    process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
    current_app.logger.info(
@@ -63,67 +70,55 @@ def create_nightly_billing_for_day(process_day):

@notify_celery.task(name="create-nightly-notification-status")
@cronitor("create-nightly-notification-status")
+@statsd(namespace="tasks")
 def create_nightly_notification_status():
-    """
-    Aggregate notification statuses into rows in ft_notification_status.
-    In order to minimise effort, this task assumes that:
-
-      - Email + SMS statuses don't change after 3 days. This is currently true
-        because all outstanding email / SMS are "timed out" after 3 days, and
-        we reject delivery receipts after this point.
-
-      - Letter statuses don't change after 9 days. There's no "timeout" for
-        letters but this is the longest we've had to cope with in the past - due
-        to major issues with our print provider.
-
-    Because the time range of the task exceeds the minimum possible retention
-    period (3 days), we need to choose which table to query for each service.
-
-    The aggregation happens for 1 extra day in case:
-
-      - This task or the "timeout" task fails to run.
-
-      - Data is (somehow) still in transit to the history table, which would
-        mean the aggregated results are temporarily incorrect.
-    """
-
+    current_app.logger.info("create-nightly-notification-status task: started")
    yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)

-    for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
-        days = 10 if notification_type == LETTER_TYPE else 4
-
-        for i in range(days):
-            process_day = yesterday - timedelta(days=i)
-
-            relevant_service_ids = get_service_ids_with_notifications_on_date(
-                notification_type, process_day
+    # email and sms
+    for i in range(4):
+        process_day = yesterday - timedelta(days=i)
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
            )
-
-            for service_id in relevant_service_ids:
-                create_nightly_notification_status_for_service_and_day.apply_async(
-                    kwargs={
-                        'process_day': process_day.isoformat(),
-                        'notification_type': notification_type,
-                        'service_id': service_id,
-                    },
-                    queue=QueueNames.REPORTING
-                )
+            current_app.logger.info(
+                f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
+                f"for type {notification_type} for {process_day}"
+            )
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
+    for i in range(10):
+        process_day = yesterday - timedelta(days=i)
+        create_nightly_notification_status_for_day.apply_async(
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )
+        current_app.logger.info(
+            f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
+            f"for type letter for {process_day}"
+        )


-@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
-def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
+@notify_celery.task(name="create-nightly-notification-status-for-day")
+@statsd(namespace="tasks")
+def create_nightly_notification_status_for_day(process_day, notification_type):
    process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: started'
+    )

    start = datetime.utcnow()
-    update_fact_notification_status(
-        process_day=process_day,
-        notification_type=notification_type,
-        service_id=service_id
-    )
-
+    transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
    end = datetime.utcnow()
    current_app.logger.info(
-        f'create-nightly-notification-status-for-service-and-day task update '
-        f'for {service_id}, {notification_type} for {process_day}: '
-        f'updated in {(end - start).seconds} seconds'
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
+        f'data fetched in {(end - start).seconds} seconds'
+    )
+
+    update_fact_notification_status(transit_data, process_day, notification_type)
+
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
+        f'task complete - {len(transit_data)} rows updated'
    )
--- a/app/celery/research_mode_tasks.py
+++ b/app/celery/research_mode_tasks.py
@@ -1,16 +1,17 @@
-import json
 import random
 from datetime import datetime, timedelta
+import json

 from flask import current_app
+from requests import request, RequestException, HTTPError
+
 from notifications_utils.s3 import s3upload
-from requests import HTTPError, request

 from app import notify_celery
 from app.aws.s3 import file_exists
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
 from app.models import SMS_TYPE
+from app.config import QueueNames
+from app.celery.process_ses_receipts_tasks import process_ses_results

 temp_fail = "7700900003"
 perm_fail = "7700900002"
@@ -65,14 +66,16 @@ def make_request(notification_type, provider, data, headers):
            timeout=60
        )
        response.raise_for_status()
-    except HTTPError as e:
+    except RequestException as e:
+        api_error = HTTPError(e)
        current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
+            "API {} request on {} failed with {}".format(
+                "POST",
                api_call,
-                e.response.status_code
+                api_error.response
            )
        )
-        raise e
+        raise api_error
    finally:
        current_app.logger.info("Mocked provider callback request finished")
    return response.json()
@@ -124,7 +127,7 @@ def create_fake_letter_response_file(self, reference):
    dvla_response_data = '{}|Sent|0|Sorted'.format(reference)

    # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
-    for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related
+    for i in sorted(range(30), key=lambda _: random.random()):
        upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
        if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
            break
@@ -164,7 +167,7 @@ def ses_notification_callback(reference):
            'processingTimeMillis': 2003,
            'recipients': ['success@simulator.amazonses.com'],
            'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
            'smtpResponse': '250 2.6.0 Message received',
            'timestamp': '2017-11-17T12:14:03.646Z'
        },
@@ -201,7 +204,7 @@ def ses_notification_callback(reference):
            'messageId': reference,
            'sendingAccountId': '12341234',
            'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
            'sourceIp': '0.0.0.1',
            'timestamp': '2017-11-17T12:14:01.643Z'
        },
@@ -211,14 +214,14 @@ def ses_notification_callback(reference):
    return {
        'Type': 'Notification',
        'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
        'Subject': None,
        'Message': json.dumps(ses_message_body),
        'Timestamp': '2017-11-17T12:14:03.710Z',
        'SignatureVersion': '1',
        'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
        'MessageAttributes': {}
    }

@@ -244,7 +247,7 @@ def _ses_bounce_callback(reference, bounce_type):
            }],
            'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
            'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
            'timestamp': '2017-11-17T12:14:05.131Z'
        },
        'mail': {
@@ -280,7 +283,7 @@ def _ses_bounce_callback(reference, bounce_type):
            'messageId': reference,
            'sendingAccountId': '12341234',
            'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
            'sourceIp': '0.0.0.1',
            'timestamp': '2017-11-17T12:14:03.000Z'
        },
@@ -289,13 +292,13 @@ def _ses_bounce_callback(reference, bounce_type):
    return {
        'Type': 'Notification',
        'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
        'Subject': None,
        'Message': json.dumps(ses_message_body),
        'Timestamp': '2017-11-17T12:14:05.149Z',
        'SignatureVersion': '1',
        'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
        'MessageAttributes': {}
    }
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -1,66 +1,54 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)

 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from sqlalchemy import between
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import and_
 from sqlalchemy.exc import SQLAlchemyError

-from app import db, notify_celery, zendesk_client
-from app.aws import s3
-from app.celery.broadcast_message_tasks import trigger_link_test
-from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app import notify_celery, zendesk_client
 from app.celery.tasks import (
-    get_recipient_csv_and_template_and_sender_id,
-    process_incomplete_jobs,
    process_job,
+    get_recipient_csv_and_template_and_sender_id,
    process_row,
-)
-from app.config import QueueNames, TaskNames
-from app.dao.invited_org_user_dao import (
-    delete_org_invitations_created_more_than_two_days_ago,
-)
-from app.dao.invited_user_dao import (
-    delete_invitations_created_more_than_two_days_ago,
-)
+    process_incomplete_jobs)
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app.config import QueueNames
+from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
    dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
    find_jobs_with_missing_rows,
-    find_missing_row_for_job,
+    find_missing_row_for_job
 )
+from app.dao.jobs_dao import dao_update_job
 from app.dao.notifications_dao import (
-    dao_old_letters_with_created_status,
-    dao_precompiled_letters_still_pending_virus_check,
-    is_delivery_slow_for_providers,
-    letters_missing_from_sending_bucket,
    notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket,
+    is_delivery_slow_for_providers,
 )
 from app.dao.provider_details_dao import (
-    dao_adjust_provider_priority_back_to_resting_points,
    dao_reduce_sms_provider_priority,
-)
-from app.dao.services_dao import (
-    dao_find_services_sending_to_tv_numbers,
-    dao_find_services_with_high_failure_rates,
+    dao_adjust_provider_priority_back_to_resting_points
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.letters.utils import generate_letter_pdf_filename
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
 from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
-    BroadcastMessage,
-    BroadcastStatusType,
    Job,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_ERROR,
+    SMS_TYPE,
+    EMAIL_TYPE,
 )
 from app.notifications.process_notifications import send_notification_to_queue


@notify_celery.task(name="run-scheduled-jobs")
+@statsd(namespace="tasks")
 def run_scheduled_jobs():
    try:
        for job in dao_set_scheduled_jobs_to_pending():
@@ -72,6 +60,7 @@ def run_scheduled_jobs():


@notify_celery.task(name="delete-verify-codes")
+@statsd(namespace="tasks")
 def delete_verify_codes():
    try:
        start = datetime.utcnow()
@@ -85,6 +74,7 @@ def delete_verify_codes():


@notify_celery.task(name="delete-invitations")
+@statsd(namespace="tasks")
 def delete_invitations():
    try:
        start = datetime.utcnow()
@@ -99,6 +89,7 @@ def delete_invitations():


@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
 def switch_current_sms_provider_on_slow_delivery():
    """
    Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
@@ -121,42 +112,32 @@ def switch_current_sms_provider_on_slow_delivery():


@notify_celery.task(name='tend-providers-back-to-middle')
+@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
    dao_adjust_provider_priority_back_to_resting_points()


@notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
 def check_job_status():
    """
    every x minutes do this check
    select
    from jobs
    where job_status == 'in progress'
-    and processing started between 30 and 35 minutes ago
-    OR where the job_status == 'pending'
-    and the job scheduled_for timestamp is between 30 and 35 minutes ago.
+    and template_type in ('sms', 'email')
+    and scheduled_at or created_at is older that 30 minutes.
    if any results then
-        update the job_status to 'error'
+        raise error
        process the rows in the csv that are missing (in another task) just do the check here.
    """
    thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
    thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)

-    incomplete_in_progress_jobs = Job.query.filter(
+    jobs_not_complete_after_30_minutes = Job.query.filter(
        Job.job_status == JOB_STATUS_IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-    incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
-        Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-
-    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
-        incomplete_pending_jobs
-    ).order_by(
-        Job.processing_started, Job.scheduled_for
-    ).all()
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
+    ).order_by(Job.processing_started).all()

    # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
    # if they haven't been re-processed in time.
@@ -175,6 +156,7 @@ def check_job_status():


@notify_celery.task(name='replay-created-notifications')
+@statsd(namespace="tasks")
 def replay_created_notifications():
    # if the notification has not be send after 1 hour, then try to resend.
    resend_created_notifications_older_than = (60 * 60)
@@ -207,72 +189,47 @@ def replay_created_notifications():
            get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)


-@notify_celery.task(name='check-if-letters-still-pending-virus-check')
-def check_if_letters_still_pending_virus_check():
-    letters = []
-
-    for letter in dao_precompiled_letters_still_pending_virus_check():
-        # find letter in the scan bucket
-        filename = generate_letter_pdf_filename(
-            letter.reference,
-            letter.created_at,
-            ignore_folder=True,
-            postage=letter.postage
-        )
-
-        if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename):
-            current_app.logger.warning(
-                f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.'
-            )
-            notify_celery.send_task(
-                name=TaskNames.SCAN_FILE,
-                kwargs={'filename': filename},
-                queue=QueueNames.ANTIVIRUS,
-            )
-        else:
-            letters.append(letter)
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()

    if len(letters) > 0:
        letter_ids = [(str(letter.id), letter.reference) for letter in letters]

-        msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes.
-            We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off
-            again or move them to technical failure.
+        msg = """{} precompiled letters have been pending-virus-check for over 90 minutes. Follow runbook to resolve:
+            https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-letter-pending-virus-scan-for-90-minutes.
+            Notifications: {}""".format(len(letters), sorted(letter_ids))

-            Notifications: {sorted(letter_ids)}"""
+        current_app.logger.exception(msg)

        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
                message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)


-@notify_celery.task(name='check-if-letters-still-in-created')
-def check_if_letters_still_in_created():
+@notify_celery.task(name='check-templated-letter-state')
+@statsd(namespace="tasks")
+def check_templated_letter_state():
    letters = dao_old_letters_with_created_status()

    if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
        msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Follow runbook to resolve: " \
-              "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
-              "#deal-with-Letters-still-in-created.".format(len(letters))
+              "Notifications: {}".format(len(letters), letter_ids)
+
+        current_app.logger.exception(msg)

        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
                message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)


@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
@@ -289,6 +246,7 @@ def check_for_missing_rows_in_completed_jobs():


@notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
+@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
    start_date = (datetime.utcnow() - timedelta(days=1))
    end_date = datetime.utcnow()
@@ -326,44 +284,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
            message += ("\nYou can find instructions for this ticket in our manual:\n"
                        "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
+            zendesk_client.create_ticket(
+                subject="[{}] High failure rates for sms spotted for services".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']
+                ),
                message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True
+                ticket_type=zendesk_client.TYPE_INCIDENT
            )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-@notify_celery.task(name='trigger-link-tests')
-def trigger_link_tests():
-    if current_app.config['CBC_PROXY_ENABLED']:
-        for cbc_name in current_app.config['ENABLED_CBCS']:
-            trigger_link_test.apply_async(kwargs={'provider': cbc_name}, queue=QueueNames.BROADCASTS)
-
-
-@notify_celery.task(name='auto-expire-broadcast-messages')
-def auto_expire_broadcast_messages():
-    expired_broadcasts = BroadcastMessage.query.filter(
-        BroadcastMessage.finishes_at <= datetime.now(),
-        BroadcastMessage.status == BroadcastStatusType.BROADCASTING,
-    ).all()
-
-    for broadcast in expired_broadcasts:
-        broadcast.status = BroadcastStatusType.COMPLETED
-
-    db.session.commit()
-
-    if expired_broadcasts:
-        notify_celery.send_task(
-            name=TaskNames.PUBLISH_GOVUK_ALERTS,
-            queue=QueueNames.GOVUK_ALERTS
-        )
-
-
-@notify_celery.task(name='remove-yesterdays-planned-tests-on-govuk-alerts')
-def remove_yesterdays_planned_tests_on_govuk_alerts():
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -1,14 +1,22 @@
 import json

 from flask import current_app
-from requests import HTTPError, RequestException, request
+from notifications_utils.statsd_decorators import statsd
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)

-from app import encryption, notify_celery
+from app import (
+    notify_celery,
+    encryption
+)
 from app.config import QueueNames
-from app.utils import DATETIME_FORMAT


@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_delivery_status_to_service(
    self, notification_id, encrypted_status_update
 ):
@@ -22,11 +30,8 @@ def send_delivery_status_to_service(
        "created_at": status_update['notification_created_at'],
        "completed_at": status_update['notification_updated_at'],
        "sent_at": status_update['notification_sent_at'],
-        "notification_type": status_update['notification_type'],
-        "template_id": status_update['template_id'],
-        "template_version": status_update['template_version']
+        "notification_type": status_update['notification_type']
    }
-
    _send_data_to_service_callback_api(
        self,
        data,
@@ -37,6 +42,7 @@ def send_delivery_status_to_service(


@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
    complaint = encryption.decrypt(complaint_data)

@@ -68,7 +74,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                'Content-Type': 'application/json',
                'Authorization': 'Bearer {}'.format(token)
            },
-            timeout=5
+            timeout=60
        )
        current_app.logger.info('{} sending {} to {}, response {}'.format(
            function_name,
@@ -79,16 +85,16 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
        response.raise_for_status()
    except RequestException as e:
        current_app.logger.warning(
-            "{} request failed for notification_id: {} and url: {}. exception: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
                function_name,
                notification_id,
                service_callback_url,
                e
            )
        )
-        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
            try:
-                self.retry(queue=QueueNames.CALLBACKS_RETRY)
+                self.retry(queue=QueueNames.RETRY)
            except self.MaxRetriesExceededError:
                current_app.logger.warning(
                    "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
@@ -97,24 +103,15 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                        notification_id
                    )
                )
-        else:
-            current_app.logger.warning(
-                "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name,
-                    notification_id,
-                    service_callback_url,
-                    e
-                )
-            )


 def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import DATETIME_FORMAT, encryption
    data = {
        "notification_id": str(notification.id),
        "notification_client_reference": notification.client_reference,
        "notification_to": notification.to,
        "notification_status": notification.status,
-        "notification_provider_response": notification.provider_response, # TODO do we have a test for provider_response
        "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
        "notification_updated_at":
            notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
@@ -122,13 +119,12 @@ def create_delivery_status_callback_data(notification, service_callback_api):
        "notification_type": notification.notification_type,
        "service_callback_api_url": service_callback_api.url,
        "service_callback_api_bearer_token": service_callback_api.bearer_token,
-        "template_id": str(notification.template_id),
-        "template_version": notification.template_version,
    }
    return encryption.encrypt(data)


 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
    data = {
        "complaint_id": str(complaint.id),
        "notification_id": str(notification.id),
--- a/app/celery/tasks.py
+++ b/app/celery/tasks.py
@@ -1,38 +1,49 @@
 import json
-from collections import defaultdict, namedtuple
 from datetime import datetime
+from collections import namedtuple, defaultdict

 from flask import current_app
-from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.columns import Columns
 from notifications_utils.postal_address import PostalAddress
 from notifications_utils.recipients import RecipientCSV
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
-from requests import HTTPError, RequestException, request
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError

-from app import create_random_identifier, create_uuid, encryption, notify_celery
+from app import (
+    create_uuid,
+    create_random_identifier,
+    DATETIME_FORMAT,
+    encryption,
+    notify_celery,
+)
 from app.aws import s3
-from app.celery import letters_pdf_tasks, provider_tasks, research_mode_tasks
+from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
 from app.config import QueueNames
-from app.dao.daily_sorted_letter_dao import (
-    dao_create_or_update_daily_sorted_letter,
-)
+from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
-from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id,
+)
 from app.dao.notifications_dao import (
-    dao_get_last_notification_added_for_job_id,
-    dao_get_notification_history_by_reference,
-    dao_update_notifications_by_reference,
    get_notification_by_id,
+    dao_update_notifications_by_reference,
+    dao_get_last_notification_added_for_job_id,
    update_notification_status_by_reference,
+    dao_get_notification_or_history_by_reference,
 )
-from app.dao.provider_details_dao import (
-    get_provider_details_by_notification_type,
-)
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
 from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
@@ -46,22 +57,19 @@ from app.models import (
    LETTER_TYPE,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
-    NOTIFICATION_RETURNED_LETTER,
    NOTIFICATION_SENDING,
-    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
    SMS_TYPE,
    DailySortedLetter,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
-from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
-from app.v2.errors import TooManyRequestsError


@notify_celery.task(name="process-job")
+@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
    start = datetime.utcnow()
    job = dao_get_job_by_id(job_id)
@@ -72,10 +80,6 @@ def process_job(job_id, sender_id=None):

    service = job.service

-    job.job_status = JOB_STATUS_IN_PROGRESS
-    job.processing_started = start
-    dao_update_job(job)
-
    if not service.active:
        job.job_status = JOB_STATUS_CANCELLED
        dao_update_job(job)
@@ -86,6 +90,10 @@ def process_job(job_id, sender_id=None):
    if __sending_limits_for_job_exceeded(service, job, job_id):
        return

+    job.job_status = JOB_STATUS_IN_PROGRESS
+    job.processing_started = start
+    dao_update_job(job)
+
    recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)

    current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
@@ -160,13 +168,9 @@ def process_row(row, template, job, service, sender_id=None):


 def __sending_limits_for_job_exceeded(service, job, job_id):
-    try:
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-        if total_sent + job.notification_count > service.message_limit:
-            raise TooManyRequestsError(service.message_limit)
-        else:
-            return False
-    except TooManyRequestsError:
+    total_sent = fetch_todays_total_message_count(service.id)
+
+    if total_sent + job.notification_count > service.message_limit:
        job.job_status = 'sending limits exceeded'
        job.processing_finished = datetime.utcnow()
        dao_update_job(job)
@@ -175,26 +179,24 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
                job_id, job.notification_count, service.message_limit)
        )
        return True
+    return False


@notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_sms(self,
             service_id,
             notification_id,
             encrypted_notification,
             sender_id=None):
    notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    if sender_id:
        reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
    else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()

    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
        current_app.logger.debug(
@@ -236,6 +238,7 @@ def save_sms(self,


@notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_email(self,
               service_id,
               notification_id,
@@ -243,17 +246,13 @@ def save_email(self,
               sender_id=None):
    notification = encryption.decrypt(encrypted_notification)

-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    if sender_id:
        reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
    else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()

    if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
        current_app.logger.info("Email {} failed as restricted service".format(notification_id))
@@ -287,19 +286,21 @@ def save_email(self,


@notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_api_email(self, encrypted_notification):

    save_api_email_or_sms(self, encrypted_notification)


@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_api_sms(self, encrypted_notification):
    save_api_email_or_sms(self, encrypted_notification)


 def save_api_email_or_sms(self, encrypted_notification):
    notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification['service_id'])
+    service = dao_fetch_service_by_id(notification['service_id'])
    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
        else provider_tasks.deliver_sms
@@ -338,10 +339,11 @@ def save_api_email_or_sms(self, encrypted_notification):
        try:
            self.retry(queue=QueueNames.RETRY)
        except self.MaxRetriesExceededError:
-            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
+            current_app.logger.error('Max retry failed' + f"Failed to persist notification {notification['id']}")


@notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_letter(
        self,
        service_id,
@@ -351,15 +353,11 @@ def save_letter(
    notification = encryption.decrypt(encrypted_notification)

    postal_address = PostalAddress.from_personalisation(
-        InsensitiveDict(notification['personalisation'])
+        Columns(notification['personalisation'])
    )

-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

    try:
        # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
@@ -380,8 +378,7 @@ def save_letter(
            job_row_number=notification['row_number'],
            notification_id=notification_id,
            reference=create_random_identifier(),
-            client_reference=get_reference_from_personalisation(notification['personalisation']),
-            reply_to_text=template.reply_to_text,
+            reply_to_text=template.get_reply_to_text(),
            status=status
        )

@@ -404,6 +401,7 @@ def save_letter(


@notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):
    # This task will be called by the FTP app to update notifications as sent to DVLA
    provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
@@ -422,6 +420,7 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):


@notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
    # This task will be called by the FTP app to update notifications as sent to DVLA

@@ -449,7 +448,6 @@ def handle_exception(task, notification, notification_id, exc):
        # Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
        # SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
        # send to the retry queue.
-        # This probably (hopefully) is not an issue with Redis as the celery backing store
        current_app.logger.exception('Retry' + retry_msg)
        try:
            task.retry(queue=QueueNames.RETRY, exc=exc)
@@ -458,6 +456,7 @@ def handle_exception(task, notification, notification_id, exc):


@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
 def update_letter_notifications_statuses(self, filename):
    notification_updates = parse_dvla_file(filename)

@@ -474,6 +473,7 @@ def update_letter_notifications_statuses(self, filename):


@notify_celery.task(bind=True, name="record-daily-sorted-counts")
+@statsd(namespace="tasks")
 def record_daily_sorted_counts(self, filename):
    sorted_letter_counts = defaultdict(int)
    notification_updates = parse_dvla_file(filename)
@@ -548,7 +548,7 @@ def update_letter_notification(filename, temporary_failures, update):


 def check_billable_units(notification_update):
-    notification = dao_get_notification_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_or_history_by_reference(notification_update.reference)

    if int(notification_update.page_count) != notification.billable_units:
        msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(
@@ -560,6 +560,7 @@ def check_billable_units(notification_update):


@notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
    inbound_api = get_service_inbound_api_for_service(service_id=service_id)
    if not inbound_api:
@@ -588,32 +589,33 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
            },
            timeout=60
        )
-        current_app.logger.debug(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
-            f"response {response.status_code}"
-        )
+        current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
+            inbound_sms_id,
+            inbound_api.url,
+            response.status_code
+        ))
        response.raise_for_status()
    except RequestException as e:
        current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
-            f"and url: {inbound_api.url}. exception: {e}"
+            "send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
+                service_id,
+                inbound_sms_id,
+                inbound_api.url,
+                e
+            )
        )
        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
            try:
                self.retry(queue=QueueNames.RETRY)
            except self.MaxRetriesExceededError:
                current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of" +
+                    f"Retry: send_inbound_sms_to_service has retried the max number of" +
                    f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
                )
-        else:
-            current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
-                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
-            )


@notify_celery.task(name='process-incomplete-jobs')
+@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
    jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]

@@ -650,6 +652,7 @@ def process_incomplete_job(job_id):


@notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
 def process_returned_letters_list(notification_references):
    updated, updated_history = dao_update_notifications_by_reference(
        notification_references,
--- a/app/clients/init.py
+++ b/app/clients/init.py
@@ -1,6 +1,3 @@
-from celery import current_app
-
-
 class ClientException(Exception):
    '''
    Base Exceptions for sending notifications that fail
@@ -20,7 +17,7 @@ STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'


-class NotificationProviderClients(object):
+class Clients(object):
    sms_clients = {}
    email_clients = {}

@@ -38,7 +35,7 @@ class NotificationProviderClients(object):
        return self.email_clients.get(name)

    def get_client_by_name_and_type(self, name, notification_type):
-        assert notification_type in ['email', 'sms'] # nosec B101
+        assert notification_type in ['email', 'sms']

        if notification_type == 'email':
            return self.get_email_client(name)
--- a/app/clients/cbc_proxy.py
+++ b/app/clients/cbc_proxy.py
@@ -1,15 +1,6 @@
 import json
-import uuid
-from abc import ABC, abstractmethod

 import boto3
-import botocore
-from flask import current_app
-from notifications_utils.template import non_gsm_characters
-from sqlalchemy.schema import Sequence
-
-from app.config import BroadcastProvider
-from app.utils import DATETIME_FORMAT, format_sequential_number

 # The variable names in this file have specific meaning in a CAP message
 #
@@ -18,282 +9,82 @@ from app.utils import DATETIME_FORMAT, format_sequential_number
 # headline is a field which we are not sure if we will use
 #
 # description is the body of the message
-
-# areas is a list of dicts, with the following items
-# * description is a string which populates the areaDesc field
-# * polygon is a list of lat/long pairs
 #
-# previous_provider_messages is a list of previous events (models.py::BroadcastProviderMessage)
-# ie a Cancel message would have a unique event but have the event of
-#    the preceeding Alert message in the previous_provider_messages field
+# references is a whitespace separated list of message identifiers
+# where each identifier is a previous sent message
+# ie a Cancel message would have a unique identifier but have the identifier of
+#    the preceeding Alert message in the references field


-class CBCProxyRetryableException(Exception):
-    pass
-
-
-class CBCProxyClient:
-    _lambda_client = None
+# Noop = no operation
+class CBCProxyNoopClient:

    def init_app(self, app):
-        if app.config.get('CBC_PROXY_ENABLED'):
-            self._lambda_client = boto3.client(
-                'lambda',
-                region_name='us-west-2',
-                aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
-                aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
-            )
-
-    def get_proxy(self, provider):
-        proxy_classes = {
-            BroadcastProvider.EE: CBCProxyEE,
-            BroadcastProvider.THREE: CBCProxyThree,
-            BroadcastProvider.O2: CBCProxyO2,
-            BroadcastProvider.VODAFONE: CBCProxyVodafone,
-        }
-        return proxy_classes[provider](self._lambda_client)
-
-
-class CBCProxyClientBase(ABC):
-    @property
-    @abstractmethod
-    def lambda_name(self):
        pass

-    @property
-    @abstractmethod
-    def failover_lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_ENGLISH(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_WELSH(self):
-        pass
-
-    def __init__(self, lambda_client):
-        self._lambda_client = lambda_client
-
-    def send_link_test(self):
-        self._send_link_test(self.lambda_name)
-        self._send_link_test(self.failover_lambda_name)
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ): pass
-
    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
+        self,
+        identifier, headline, description,
    ):
        pass

    # We have not implementated updating a broadcast
    def update_and_send_broadcast(
        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, channel, message_number=None
+        identifier, references, headline, description,
    ):
        pass

+    # We have not implemented cancelling a broadcast
    def cancel_broadcast(
        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, message_number=None
+        identifier, references, headline, description,
    ):
        pass

-    def _invoke_lambda_with_failover(self, payload):
-        result = self._invoke_lambda(self.lambda_name, payload)

-        if not result:
-            failover_result = self._invoke_lambda(self.failover_lambda_name, payload)
-            if not failover_result:
-                raise CBCProxyRetryableException(
-                    f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}'
-                )
+class CBCProxyClient:

-        return result
+    def init_app(self, app):
+        self._lambda_client = boto3.client(
+            'lambda',
+            region_name='eu-west-2',
+            aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
+            aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
+        )

-    def _invoke_lambda(self, lambda_name, payload):
-        payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        try:
-            current_app.logger.info(
-                f"Calling lambda {lambda_name} with payload {str(payload)[:1000]}"
-            )
+    def create_and_send_broadcast(
+        self,
+        identifier, headline, description,
+    ):
+        payload_bytes = bytes(json.dumps({
+            'identifier': identifier,
+            'headline': headline,
+            'description': description,
+        }), encoding='utf8')

-            result = self._lambda_client.invoke(
-                FunctionName=lambda_name,
-                InvocationType='RequestResponse',
-                Payload=payload_bytes,
-            )
-        except botocore.exceptions.ClientError:
-            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
-            success = False
-            return success
+        result = self._lambda_client.invoke(
+            FunctionName='bt-ee-1-proxy',
+            InvocationType='RequestResponse',
+            Payload=payload_bytes,
+        )

        if result['StatusCode'] > 299:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with status code { result['StatusCode']}, {result.get('Payload')}"
-            )
-            success = False
+            raise Exception('Could not invoke lambda')

-        elif 'FunctionError' in result:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with function error { result['Payload'].read() }"
-            )
-            success = False
+        if 'FunctionError' in result:
+            raise Exception('Function exited with unhandled exception')

-        else:
-            success = True
-
-        return success
-
-    def infer_language_from(self, content):
-        if non_gsm_characters(content):
-            return self.LANGUAGE_WELSH
-        return self.LANGUAGE_ENGLISH
-
-
-class CBCProxyOne2ManyClient(CBCProxyClientBase):
-    LANGUAGE_ENGLISH = 'en-GB'
-    LANGUAGE_WELSH = 'cy-GB'
-
-    def _send_link_test(
+    # We have not implementated updating a broadcast
+    def update_and_send_broadcast(
        self,
-        lambda_name,
+        identifier, references, headline, description,
    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_format': 'cap'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_format': 'cap',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
+        pass

+    # We have not implemented cancelling a broadcast
    def cancel_broadcast(
        self,
-        identifier, previous_provider_messages,
-        sent, message_number=None
+        identifier, references, headline, description,
    ):
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_format': 'cap',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-
-class CBCProxyEE(CBCProxyOne2ManyClient):
-    lambda_name = 'ee-1-proxy'
-    failover_lambda_name = 'ee-2-proxy'
-
-
-class CBCProxyThree(CBCProxyOne2ManyClient):
-    lambda_name = 'three-1-proxy'
-    failover_lambda_name = 'three-2-proxy'
-
-
-class CBCProxyO2(CBCProxyOne2ManyClient):
-    lambda_name = 'o2-1-proxy'
-    failover_lambda_name = 'o2-2-proxy'
-
-
-class CBCProxyVodafone(CBCProxyClientBase):
-    lambda_name = 'vodafone-1-proxy'
-    failover_lambda_name = 'vodafone-2-proxy'
-
-    LANGUAGE_ENGLISH = 'English'
-    LANGUAGE_WELSH = 'Welsh'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        from app import db
-        sequence = Sequence('broadcast_provider_message_number_seq')
-        sequential_number = db.session.connection().execute(sequence)
-        formatted_seq_number = format_sequential_number(sequential_number)
-
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_number': formatted_seq_number,
-            'message_format': 'ibag'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, message_number, headline, description, areas, sent, expires, channel
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self, identifier, previous_provider_messages, sent, message_number
-    ):
-
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "message_number": format_sequential_number(message.message_number),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
+        pass
--- a/app/clients/document_download.py
+++ b/app/clients/document_download.py
@@ -1,4 +1,5 @@
 import requests
+
 from flask import current_app


@@ -41,9 +42,7 @@ class DocumentDownloadClient:
            # if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
            # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
            # unrecognised) should be raised as a 500 internal server error here.
-            if e.response is None:
-                raise Exception(f'Unhandled document download error: {repr(e)}')
-            elif e.response.status_code == 400:
+            if e.response.status_code == 400:
                error = DocumentDownloadError.from_exception(e)
                current_app.logger.info(
                    'Document download request failed with error: {}'.format(error.message)
--- a/app/clients/email/init.py
+++ b/app/clients/email/init.py
@@ -1,4 +1,4 @@
-from app.clients import Client, ClientException
+from app.clients import ClientException, Client


 class EmailClientException(ClientException):
@@ -8,18 +8,6 @@ class EmailClientException(ClientException):
    pass


-class EmailClientNonRetryableException(ClientException):
-    '''
-    Represents an error returned from the email client API with a 4xx response code
-    that should not be retried and should instead be marked as technical failure.
-    An example of this would be an email address that makes it through our
-    validation rules but is rejected by SES. There is no point in retrying this type as
-    it will always fail however many calls to SES. Whereas a throttling error would not
-    use this exception as it may succeed if we retry
-    '''
-    pass
-
-
 class EmailClient(Client):
    '''
    Base Email client for sending emails.
@@ -28,6 +16,5 @@ class EmailClient(Client):
    def send_email(self, *args, **kwargs):
        raise NotImplementedError('TODO Need to implement.')

-    @property
-    def name(self):
+    def get_name(self):
        raise NotImplementedError('TODO Need to implement.')
--- a/app/clients/email/aws_ses.py
+++ b/app/clients/email/aws_ses.py
@@ -1,15 +1,11 @@
-from time import monotonic
-
 import boto3
 import botocore
 from flask import current_app
+from time import monotonic
+from notifications_utils.recipients import InvalidEmailError

 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
-from app.clients.email import (
-    EmailClient,
-    EmailClientException,
-    EmailClientNonRetryableException,
-)
+from app.clients.email import (EmailClientException, EmailClient)

 ses_response_map = {
    'Permanent': {
@@ -59,11 +55,27 @@ class AwsSesClient(EmailClient):
    def init_app(self, region, statsd_client, *args, **kwargs):
        self._client = boto3.client('ses', region_name=region)
        super(AwsSesClient, self).__init__(*args, **kwargs)
+        self.name = 'ses'
        self.statsd_client = statsd_client

-    @property
-    def name(self):
-        return 'ses'
+        # events are generally undocumented, but some that might be of interest are:
+        # before-call, after-call, after-call-error, request-created, response-received
+        self._client.meta.events.register(f'request-created.ses.SendEmail', self.ses_request_created_hook)
+        self._client.meta.events.register(f'response-received.ses.SendEmail', self.ses_response_received_hook)
+
+    def ses_request_created_hook(self, **kwargs):
+        # request created may be called multiple times if the request auto-retries. We want to count all these as the
+        # same request for timing purposes, so only reset the start time if it was cleared completely
+        if self.ses_start_time == 0:
+            self.ses_start_time = monotonic()
+
+    def ses_response_received_hook(self, **kwargs):
+        # response received may be called multiple times if the request auto-retries, however, we want to count the last
+        # time it triggers for timing purposes, so always reset the elapsed time
+        self.ses_elapsed_time = monotonic() - self.ses_start_time
+
+    def get_name(self):
+        return self.name

    def send_email(self,
                   source,
@@ -72,6 +84,8 @@ class AwsSesClient(EmailClient):
                   body,
                   html_body='',
                   reply_to_address=None):
+        self.ses_elapsed_time = 0
+        self.ses_start_time = 0
        try:
            if isinstance(to_addresses, str):
                to_addresses = [to_addresses]
@@ -108,7 +122,10 @@ class AwsSesClient(EmailClient):

            # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
            if e.response['Error']['Code'] == 'InvalidParameterValue':
-                raise EmailClientNonRetryableException(e.response['Error']['Message'])
+                raise InvalidEmailError('email: "{}" message: "{}"'.format(
+                    to_addresses[0],
+                    e.response['Error']['Message']
+                ))
            elif (
                e.response['Error']['Code'] == 'Throttling'
                and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
@@ -124,6 +141,8 @@ class AwsSesClient(EmailClient):
            elapsed_time = monotonic() - start_time
            current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
            self.statsd_client.timing("clients.ses.request-time", elapsed_time)
+            if self.ses_elapsed_time != 0:
+                self.statsd_client.timing("clients.ses.raw-request-time", self.ses_elapsed_time)
            self.statsd_client.incr("clients.ses.success")
            return response['MessageId']

--- a/app/clients/email/aws_ses_stub.py
+++ b/app/clients/email/aws_ses_stub.py
@@ -1,10 +1,10 @@
 import json
-from time import monotonic

 from flask import current_app
 from requests import request
+from time import monotonic

-from app.clients.email import EmailClient, EmailClientException
+from app.clients.email import (EmailClientException, EmailClient)


 class AwsSesStubClientException(EmailClientException):
@@ -13,12 +13,12 @@ class AwsSesStubClientException(EmailClientException):

 class AwsSesStubClient(EmailClient):
    def init_app(self, region, statsd_client, stub_url):
+        self.name = 'ses'
        self.statsd_client = statsd_client
        self.url = stub_url

-    @property
-    def name(self):
-        return 'ses'
+    def get_name(self):
+        return self.name

    def send_email(self,
                   source,
--- a/app/clients/performance_platform/init.py
+++ b/app/clients/performance_platform/init.py
--- a/app/clients/performance_platform/performance_platform_client.py
+++ b/app/clients/performance_platform/performance_platform_client.py
@@ -1,8 +1,9 @@
 import base64
 import json

-import requests
 from flask import current_app
+import requests
+
 from notifications_utils.timezones import convert_utc_to_bst


--- a/app/clients/sms/init.py
+++ b/app/clients/sms/init.py
@@ -1,10 +1,10 @@
-from app.clients import Client, ClientException
+from app.clients import (Client, ClientException)


 class SmsClientResponseException(ClientException):
-    """
+    '''
    Base Exception for SmsClientsResponses
-    """
+    '''

    def __init__(self, message):
        self.message = message
@@ -14,15 +14,12 @@ class SmsClientResponseException(ClientException):


 class SmsClient(Client):
-    """
+    '''
    Base Sms client for sending smss.
-    """
-
-    def init_app(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+    '''

    def send_sms(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

    def get_name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
--- a/app/clients/sms/aws_sns.py
+++ b/app/clients/sms/aws_sns.py
@@ -1,89 +0,0 @@
-import re
-from time import monotonic
-
-import boto3
-import botocore
-import phonenumbers
-
-from app.clients.sms import SmsClient
-
-
-class AwsSnsClient(SmsClient):
-    """
-    AwsSns sms client
-    """
-
-    def init_app(self, current_app, statsd_client, *args, **kwargs):
-        self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"])
-        self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"])
-        super(SmsClient, self).__init__(*args, **kwargs)
-        self.current_app = current_app
-        self.statsd_client = statsd_client
-        self.long_code_regex = re.compile(r"^\+1\d{10}$")
-        
-    @property
-    def name(self):
-        return 'sns'
-
-    def get_name(self):
-        return 'sns'
-
-    def send_sms(self, to, content, reference, sender=None, international=False):
-        matched = False
-
-        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
-            matched = True
-            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
-
-            client = self._client
-            # See documentation
-            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
-            attributes = {
-                "AWS.SNS.SMS.SMSType": {
-                    "DataType": "String",
-                    "StringValue": "Transactional",
-                }
-            }
-
-            # If sending with a long code number, we need to use another AWS region
-            # and specify the phone number we want to use as the origination number
-            send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender)
-            if send_with_dedicated_phone_number:
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": sender,
-                }
-
-            # If the number is US based, we must use a US Toll Free number to send the message
-            country = phonenumbers.region_code_for_number(match.number)
-            if country == "US":
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
-                }
-
-            try:
-                start_time = monotonic()
-                response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
-            except botocore.exceptions.ClientError as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            except Exception as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            finally:
-                elapsed_time = monotonic() - start_time
-                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
-                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
-                self.statsd_client.incr("clients.sns.success")
-            return response["MessageId"]
-
-        if not matched:
-            self.statsd_client.incr("clients.sns.error")
-            self.current_app.logger.error("No valid numbers found in {}".format(to))
-            raise ValueError("No valid numbers found for SMS delivery")
-
-    def _send_with_dedicated_phone_number(self, sender):
-        return sender and re.match(self.long_code_regex, sender)
--- a/app/clients/sms/firetext.py
+++ b/app/clients/sms/firetext.py
@@ -0,0 +1,130 @@
+import json
+import logging
+
+from time import monotonic
+from requests import request, RequestException
+
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+logger = logging.getLogger(__name__)
+
+# Firetext will send a delivery receipt with three different status codes.
+# The `firetext_response` maps these codes to the notification statistics status and notification status.
+# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
+# the notification status to temporary-failure rather than permanent failure.
+#  See the code in the notification_dao.update_notifications_status_by_id
+firetext_responses = {
+    '0': 'delivered',
+    '1': 'permanent-failure',
+    '2': 'pending'
+}
+
+firetext_codes = {
+    # code '000' means 'No errors reported'
+    '101': {'status': 'permanent-failure', 'reason': 'Unknown Subscriber'},
+    '102': {'status': 'temporary-failure', 'reason': 'Absent Subscriber'},
+    '103': {'status': 'temporary-failure', 'reason': 'Subscriber Busy'},
+    '104': {'status': 'temporary-failure', 'reason': 'No Subscriber Memory'},
+    '201': {'status': 'permanent-failure', 'reason': 'Invalid Number'},
+    '301': {'status': 'permanent-failure', 'reason': 'SMS Not Supported'},
+    '302': {'status': 'temporary-failure', 'reason': 'SMS Not Supported'},
+    '401': {'status': 'permanent-failure', 'reason': 'Message Rejected'},
+    '900': {'status': 'temporary-failure', 'reason': 'Routing Error'},
+}
+
+
+def get_firetext_responses(status, detailed_status_code=None):
+    detailed_status = firetext_codes[detailed_status_code]['reason'] if firetext_codes.get(
+        detailed_status_code, None
+    ) else None
+    return (firetext_responses[status], detailed_status)
+
+
+def get_message_status_and_reason_from_firetext_code(detailed_status_code):
+    return firetext_codes[detailed_status_code]['status'], firetext_codes[detailed_status_code]['reason']
+
+
+class FiretextClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class FiretextClient(SmsClient):
+    '''
+    FireText sms client.
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('FIRETEXT_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'firetext'
+        self.url = current_app.config.get('FIRETEXT_URL')
+        self.statsd_client = statsd_client
+
+    def get_name(self):
+        return self.name
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.firetext.success")
+        else:
+            self.statsd_client.incr("clients.firetext.error")
+            self.current_app.logger.error(log_message)
+
+    def send_sms(self, to, content, reference, sender=None):
+
+        data = {
+            "apiKey": self.api_key,
+            "from": self.from_number if sender is None else sender,
+            "to": to.replace('+', ''),
+            "message": content,
+            "reference": reference
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.url,
+                data=data,
+                timeout=60
+            )
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+                if response.json()['code'] != 0:
+                    raise ValueError()
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise FiretextClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise FiretextClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
+            self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.firetext.raw-request-time", response.elapsed.total_seconds())
+        return response
--- a/app/clients/sms/mmg.py
+++ b/app/clients/sms/mmg.py
@@ -0,0 +1,140 @@
+import json
+from time import monotonic
+from requests import (request, RequestException)
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+mmg_response_map = {
+    '2': {'status': 'permanent-failure', 'substatus': {
+        "1": "Number does not exist",
+        "4": "Rejected by operator",
+        "5": "Unidentified Subscriber",
+        "9": "Undelivered",
+        "11": "Service for Subscriber suspended",
+        "12": "Illegal equipment",
+        "2049": "Subscriber IMSI blacklisted",
+        "2050": "Number blacklisted in do-not-disturb blacklist",
+        "2052": "Destination number blacklisted",
+        "2053": "Source address blacklisted"
+    }},
+    '3': {'status': 'delivered', 'substatus': {"2": "Delivered to operator", "5": "Delivered to handset"}},
+    '4': {'status': 'temporary-failure', 'substatus': {
+        "6": "Absent Subscriber",
+        "8": "Roaming not allowed",
+        "13": "SMS Not Supported",
+        "15": "Expired",
+        "27": "Absent Subscriber",
+        "29": "Invalid delivery report",
+        "32": "Delivery Failure",
+    }},
+    '5': {'status': 'permanent-failure', 'substatus': {
+        "6": "Network out of coverage",
+        "8": "Incorrect number prefix",
+        "10": "Number on do-not-disturb service",
+        "11": "Sender id not registered",
+        "13": "Sender id blacklisted",
+        "14": "Destination number blacklisted",
+        "19": "Routing unavailable",
+        "20": "Rejected by anti-flooding mechanism",
+        "21": "System error",  # it says to retry those messages or contact support
+        "23": "Duplicate message id",
+        "24": "Message formatted incorrectly",
+        "25": "Message too long",
+        "51": "Missing recipient value",
+        "52": "Invalid destination",
+    }},
+}
+
+
+def get_mmg_responses(status, detailed_status_code=None):
+    return (mmg_response_map[status]["status"], mmg_response_map[status]["substatus"].get(detailed_status_code, None))
+
+
+class MMGClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class MMGClient(SmsClient):
+    '''
+    MMG sms client
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('MMG_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'mmg'
+        self.statsd_client = statsd_client
+        self.mmg_url = current_app.config.get('MMG_URL')
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.mmg_url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.mmg.success")
+        else:
+            self.statsd_client.incr("clients.mmg.error")
+            self.current_app.logger.error(log_message)
+
+    def get_name(self):
+        return self.name
+
+    def send_sms(self, to, content, reference, multi=True, sender=None):
+        data = {
+            "reqType": "BULK",
+            "MSISDN": to,
+            "msg": content,
+            "sender": self.from_number if sender is None else sender,
+            "cid": reference,
+            "multi": multi
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.mmg_url,
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Basic {}'.format(self.api_key)
+                },
+                timeout=60
+            )
+
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise MMGClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise MMGClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.mmg.raw-request-time", response.elapsed.total_seconds())
+
+            self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
+
+        return response
--- a/app/cloudfoundry_config.py
+++ b/app/cloudfoundry_config.py
@@ -1,33 +1,21 @@
-import json
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
 import os
+import json


-def find_by_service_name(services, service_name):
-    for i in range(len(services)):
-        if services[i]['name'] == service_name:
-            return services[i]
-    return None
-
 def extract_cloudfoundry_config():
    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+    set_config_env_vars(vcap_services)

+
+def set_config_env_vars(vcap_services):
    # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
-    # Redis config
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']

-    # CSV Upload Bucket Name
-    bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
-    if bucket_service:
-        os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
-        os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
-
-    # Contact List Bucket Name
-    bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
-    if bucket_service:
-        os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
-        os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']
+    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
--- a/app/commands.py
+++ b/app/commands.py
@@ -1,33 +1,29 @@
 import csv
 import functools
-import itertools
-import os
 import uuid
 from datetime import datetime, timedelta
+from decimal import Decimal

 import click
 import flask
+import itertools
 from click_datetime import Datetime as click_dt
 from flask import current_app, json
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
-from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from notifications_utils.statsd_decorators import statsd

-from app import db
+from app import db, DATETIME_FORMAT, encryption
 from app.aws import s3
-from app.celery.letters_pdf_tasks import (
-    get_pdf_for_templated_letter,
-    resanitise_pdf,
-)
-from app.celery.tasks import process_row, record_daily_sorted_counts
+from app.celery.tasks import record_daily_sorted_counts, process_row
+from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
+from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app.celery.reporting_tasks import create_nightly_notification_status_for_day
 from app.config import QueueNames
-from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
-    set_default_free_allowance_for_service,
-)
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
 from app.dao.fact_billing_dao import (
    delete_billing_data_for_service_for_day,
    fetch_billing_data_for_day,
@@ -35,40 +31,36 @@ from app.dao.fact_billing_dao import (
    update_fact_billing,
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.dao.organisation_dao import (
-    dao_add_service_to_organisation,
-    dao_get_organisation_by_email_address,
-    dao_get_organisation_by_id,
-)
-from app.dao.permissions_dao import permission_dao
+from app.dao.organisation_dao import dao_get_organisation_by_email_address, dao_add_service_to_organisation
+
+from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.services_dao import (
+    delete_service_and_all_associated_db_objects,
    dao_fetch_all_services_by_user,
    dao_fetch_all_services_created_by_user,
    dao_fetch_service_by_id,
-    dao_update_service,
-    delete_service_and_all_associated_db_objects,
+    dao_update_service
 )
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.users_dao import (
-    delete_model_user,
-    delete_user_verify_codes,
-    get_user_by_email,
-)
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes, get_user_by_email
 from app.models import (
-    KEY_TYPE_TEST,
+    PROVIDERS,
    NOTIFICATION_CREATED,
+    KEY_TYPE_TEST,
    SMS_TYPE,
-    AnnualBilling,
-    Domain,
-    EmailBranding,
-    LetterBranding,
+    EMAIL_TYPE,
+    LETTER_TYPE,
+    User,
    Notification,
    Organisation,
-    Permission,
+    Domain,
    Service,
-    User,
+    EmailBranding,
+    LetterBranding,
 )
-from app.utils import get_london_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before


@click.group(name='command', help='Additional commands')
@@ -81,28 +73,32 @@ class notify_command:
        self.name = name

    def __call__(self, func):
-        decorators = [
-            functools.wraps(func),  # carry through function name, docstrings, etc.
-            click.command(name=self.name),  # turn it into a click.Command
-        ]
-
-        # in the test environment the app context is already provided and having
-        # another will lead to the test db connection being closed prematurely
-        if os.getenv('NOTIFY_ENVIRONMENT', '') != 'test':
-            # with_appcontext ensures the config is loaded, db connected, etc.
-            decorators.insert(0, flask.cli.with_appcontext)
-
+        # we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
+        # we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
+        # Then we need to turn it into a click.Command - that's what command_group.add_command expects.
+        @click.command(name=self.name)
+        @functools.wraps(func)
+        @flask.cli.with_appcontext
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)

-        for decorator in decorators:
-            # this syntax is equivalent to e.g. "@flask.cli.with_appcontext"
-            wrapper = decorator(wrapper)
-
        command_group.add_command(wrapper)
+
        return wrapper


+@notify_command()
+@click.option('-p', '--provider_name', required=True, type=click.Choice(PROVIDERS))
+@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals', type=float)
+@click.option('-d', '--valid_from', required=True, type=click_dt(format='%Y-%m-%dT%H:%M:%S'))
+def create_provider_rates(provider_name, cost, valid_from):
+    """
+    Backfill rates for a given provider
+    """
+    cost = Decimal(cost)
+    dao_create_provider_rates(provider_name, valid_from, cost)
+
+
@notify_command()
@click.option('-u', '--user_email_prefix', required=True, help="""
    Functional test user email prefix. eg "notify-test-preview"
@@ -151,8 +147,8 @@ def backfill_notification_statuses():
    `Notification._status_enum`
    """
    LIMIT = 250000
-    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
+    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -169,7 +165,7 @@ def update_notification_international_flag():
    """
    # 250,000 rows takes 30 seconds to update.
    subq = "select id from notifications where international is null limit 250000"
-    update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input
+    update = "update notifications set international = False where id in ({})".format(subq)
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -180,7 +176,7 @@ def update_notification_international_flag():

    # Now update notification_history
    subq_history = "select id from notification_history where international is null limit 250000"
-    update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input
+    update_history = "update notification_history set international = False where id in ({})".format(subq_history)
    result_history = db.session.execute(subq_history).fetchall()
    while len(result_history) > 0:
        db.session.execute(update_history)
@@ -201,8 +197,8 @@ def fix_notification_statuses_not_in_sync():
    """
    MAX = 10000

-    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input
-    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input
+    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
+    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
    result = db.session.execute(subq).fetchall()

    while len(result) > 0:
@@ -211,8 +207,9 @@ def fix_notification_statuses_not_in_sync():
        db.session.commit()
        result = db.session.execute(subq).fetchall()

-    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)  # nosec B608
-    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input
+    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
+        .format(MAX)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
    result = db.session.execute(subq_hist).fetchall()

    while len(result) > 0:
@@ -222,43 +219,255 @@ def fix_notification_statuses_not_in_sync():
        result = db.session.execute(subq_hist).fetchall()


+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+
+    WARNING: This does not overwrite existing data. You need to delete
+             the existing data or Performance Platform will double-count.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_processing_time(start_date, end_date):
+    """
+    Send historical processing time to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+        # because the tz conversion funcs talk about midnight, and the midnight before last,
+        # we want to pretend we're running this from the next morning, so add one.
+        process_date = start_date + timedelta(days=i + 1)
+
+        process_start_date = get_midnight_for_day_before(process_date)
+        process_end_date = get_london_midnight_in_utc(process_date)
+
+        print('Sending notification processing-time for {} - {}'.format(
+            process_start_date.isoformat(),
+            process_end_date.isoformat()
+        ))
+        send_processing_time_for_start_and_end(process_start_date, process_end_date)
+
+
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='list-routes')
+def list_routes():
+    """List URLs of all application routes."""
+    for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
+        print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
+
+
@notify_command(name='insert-inbound-numbers')
@click.option('-f', '--file_name', required=True,
              help="""Full path of the file to upload, file is a contains inbound numbers,
              one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
    print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    file = open(file_name)
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"

-        for line in file:
-            line = line.strip()
-            if line:
-                print(line)
-                db.session.execute(sql.format(uuid.uuid4(), line))
-                db.session.commit()
+    for line in file:
+        print(line)
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
+        db.session.commit()
+    file.close()


-@notify_command(name='replay-create-pdf-for-templated-letter')
+@notify_command(name='replay-create-pdf-letters')
@click.option('-n', '--notification_id', type=click.UUID, required=True,
              help="Notification id of the letter that needs the get_pdf_for_templated_letter task replayed")
-def replay_create_pdf_for_templated_letter(notification_id):
+def replay_create_pdf_letters(notification_id):
    print("Create task to get_pdf_for_templated_letter for notification: {}".format(notification_id))
    get_pdf_for_templated_letter.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)


-@notify_command(name='recreate-pdf-for-precompiled-or-uploaded-letter')
-@click.option('-n', '--notification_id', type=click.UUID, required=True,
-              help="Notification ID of the precompiled or uploaded letter")
-def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id):
-    print(f"Call resanitise_pdf task for notification: {notification_id}")
-    resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS)
+@notify_command(name='replay-service-callbacks')
+@click.option('-f', '--file_name', required=True,
+              help="""Full path of the file to upload, file is a contains client references of
+              notifications that need the status to be sent to the service.""")
+@click.option('-s', '--service_id', required=True,
+              help="""The service that the callbacks are for""")
+def replay_service_callbacks(file_name, service_id):
+    print("Start send service callbacks for service: ", service_id)
+    callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
+    if not callback_api:
+        print("Callback api was not found for service: {}".format(service_id))
+        return
+
+    errors = []
+    notifications = []
+    file = open(file_name)
+
+    for ref in file:
+        try:
+            notification = Notification.query.filter_by(client_reference=ref.strip()).one()
+            notifications.append(notification)
+        except NoResultFound:
+            errors.append("Reference: {} was not found in notifications.".format(ref))
+
+    for e in errors:
+        print(e)
+    if errors:
+        raise Exception("Some notifications for the given references were not found")
+
+    for n in notifications:
+        data = {
+            "notification_id": str(n.id),
+            "notification_client_reference": n.client_reference,
+            "notification_to": n.to,
+            "notification_status": n.status,
+            "notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
+            "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
+            "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
+            "notification_type": n.notification_type,
+            "service_callback_api_url": callback_api.url,
+            "service_callback_api_bearer_token": callback_api.bearer_token,
+        }
+        encrypted_status_update = encryption.encrypt(data)
+        send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
+                                                    queue=QueueNames.CALLBACKS)
+
+    print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
+        service_id, len(notifications)))


 def setup_commands(application):
    application.cli.add_command(command_group)


+@notify_command(name='migrate-data-to-ft-billing')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@statsd(namespace="tasks")
+def migrate_data_to_ft_billing(start_date, end_date):
+
+    current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
+
+    process_date = start_date
+    total_updated = 0
+
+    while process_date < end_date:
+        start_time = datetime.utcnow()
+        # migrate data into ft_billing, upserting the data if it the record already exists
+        sql = \
+            """
+            insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
+                international, billable_units, notifications_sent, rate, postage, created_at)
+                select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
+                    case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
+                from (
+                    select
+                        n.id,
+                        (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
+                        coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
+                        coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
+                        n.notification_type,
+                        coalesce(n.sent_by, (
+                        case
+                        when notification_type = 'sms' then
+                            coalesce(sent_by, 'unknown')
+                        when notification_type = 'letter' then
+                            coalesce(sent_by, 'dvla')
+                        else
+                            coalesce(sent_by, 'ses')
+                        end )) as provider,
+                        coalesce(n.rate_multiplier,1) as rate_multiplier,
+                        s.crown,
+                        coalesce((select rates.rate from rates
+                        where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
+                        order by rates.valid_from desc limit 1), 0) as sms_rate,
+                        coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
+                        and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
+                        and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
+                        as letter_rate,
+                        coalesce(n.international, false) as international,
+                        n.billable_units,
+                        1 as notifications_sent,
+                        coalesce(n.postage, 'none') as postage,
+                        now() as created_at
+                    from public.notification_history n
+                    left join services s on s.id = n.service_id
+                    where n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
+                        and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
+                        at time zone 'UTC'
+                        and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
+                    ) as individual_record
+                group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sms_rate, letter_rate, postage, created_at
+                order by bst_date
+            on conflict on constraint ft_billing_pkey do update set
+             billable_units = excluded.billable_units,
+             notifications_sent = excluded.notifications_sent,
+             rate = excluded.rate,
+             updated_at = now()
+            """
+
+        result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
+        db.session.commit()
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
+
+        process_date += timedelta(days=1)
+
+        total_updated += result.rowcount
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
@notify_command(name='rebuild-ft-billing-for-day')
@click.option('-s', '--service_id', required=False, type=click.UUID)
@click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -298,6 +507,29 @@ def rebuild_ft_billing_for_day(service_id, day):
            rebuild_ft_data(day, row.service_id)


+@notify_command(name='migrate-data-to-ft-notification-status')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
+@statsd(namespace="tasks")
+def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
+    notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
+
+    start_date = start_date.date()
+    end_date = end_date.date()
+    for day_diff in range((end_date - start_date).days + 1):
+        process_day = start_date + timedelta(days=day_diff)
+        for notification_type in notification_types:
+            print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
+                process_day,
+                notification_type
+            ))
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
+            )
+
+
@notify_command(name='bulk-invite-user-to-service')
@click.option('-f', '--file_name', required=True,
              help="Full path of the file containing a list of email address for people to invite to a service")
@@ -314,7 +546,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi
    #  platform_admin
    #  view_activity
    # "send_texts,send_emails,send_letters,view_activity"
-    from app.service_invite.rest import create_invited_user
+    from app.invite.rest import create_invited_user
    file = open(file_name)
    for email_address in file:
        data = {
@@ -459,14 +691,15 @@ def replay_daily_sorted_count_files(file_extension):

@notify_command(name='populate-organisations-from-file')
@click.option('-f', '--file_name', required=True,
-              help="Pipe delimited file containing organisation name, sector, argeement_signed, domains")
+              help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains")
 def populate_organisations_from_file(file_name):
    # [0] organisation name:: name of the organisation insert if organisation is missing.
-    # [1] sector:: Federal | State only
-    # [2] argeement_signed:: TRUE | FALSE
-    # [3] domains:: comma separated list of domains related to the organisation
-    # [4] email branding name: name of the default email branding for the org
-    # [5] letter branding name: name of the default letter branding for the org
+    # [1] sector:: Central | Local | NHS only
+    # [2] crown:: TRUE | FALSE only
+    # [3] argeement_signed:: TRUE | FALSE
+    # [4] domains:: comma separated list of domains related to the organisation
+    # [5] email branding name: name of the default email branding for the org
+    # [6] letter branding name: name of the default letter branding for the org

    # The expectation is that the organisation, organisation_to_service
    # and user_to_organisation will be cleared before running this command.
@@ -484,17 +717,18 @@ def populate_organisations_from_file(file_name):
            columns = line.split('|')
            print(columns)
            email_branding = None
-            email_branding_column = columns[4].strip()
+            email_branding_column = columns[5].strip()
            if len(email_branding_column) > 0:
                email_branding = EmailBranding.query.filter(EmailBranding.name == email_branding_column).one()
            letter_branding = None
-            letter_branding_column = columns[5].strip()
+            letter_branding_column = columns[6].strip()
            if len(letter_branding_column) > 0:
                letter_branding = LetterBranding.query.filter(LetterBranding.name == letter_branding_column).one()
            data = {
                'name': columns[0],
                'active': True,
-                'agreement_signed': boolean_or_none(columns[2]),
+                'agreement_signed': boolean_or_none(columns[3]),
+                'crown': boolean_or_none(columns[2]),
                'organisation_type': columns[1].lower(),
                'email_branding_id': email_branding.id if email_branding else None,
                'letter_branding_id': letter_branding.id if letter_branding else None
@@ -507,7 +741,7 @@ def populate_organisations_from_file(file_name):
            except IntegrityError:
                print("duplicate org", org.name)
                db.session.rollback()
-            domains = columns[3].split(',')
+            domains = columns[4].split(',')
            for d in domains:
                if len(d.strip()) > 0:
                    domain = Domain(domain=d.strip(), organisation_id=org.id)
@@ -519,40 +753,6 @@ def populate_organisations_from_file(file_name):
                        db.session.rollback()


-@notify_command(name='populate-organisation-agreement-details-from-file')
-@click.option('-f', '--file_name', required=True,
-              help="CSV file containing id, agreement_signed_version, "
-              "agreement_signed_on_behalf_of_name, agreement_signed_at")
-def populate_organisation_agreement_details_from_file(file_name):
-    """
-    The input file should be a comma separated CSV file with a header row and 4 columns
-    id: the organisation ID
-    agreement_signed_version
-    agreement_signed_on_behalf_of_name
-    agreement_signed_at: The date the agreement was signed in the format of 'dd/mm/yyyy'
-    """
-    with open(file_name) as f:
-        csv_reader = csv.reader(f)
-
-        # ignore the header row
-        next(csv_reader)
-
-        for row in csv_reader:
-            org = dao_get_organisation_by_id(row[0])
-
-            current_app.logger.info(f"Updating {org.name}")
-
-            if not org.agreement_signed:
-                raise RuntimeError('Agreement was not signed')
-
-            org.agreement_signed_version = float(row[1])
-            org.agreement_signed_on_behalf_of_name = row[2].strip()
-            org.agreement_signed_at = datetime.strptime(row[3], "%d/%m/%Y")
-
-            db.session.add(org)
-            db.session.commit()
-
-
@notify_command(name='get-letter-details-from-zips-sent-file')
@click.argument('file_paths', required=True, nargs=-1)
@statsd(namespace="tasks")
@@ -727,115 +927,3 @@ def process_row_from_job(job_id, job_row_number):
            notification_id = process_row(row, template, job, job.service)
            current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                job_row_number, job_id, notification_id))
-
-
-@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing_with_the_previous_years_allowance(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
-@notify_command(name='populate-annual-billing-with-defaults')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2021""")
-@click.option('-m', '--missing-services-only', default=True, type=bool,
-              help="""If true then only populate services missing from annual billing for the year.
-                      If false populate the default values for all active services.""")
-def populate_annual_billing_with_defaults(year, missing_services_only):
-    """
-    Add or update annual billing with free allowance defaults for all active services.
-    The default free allowance limits are in: app/dao/annual_billing_dao.py:57.
-
-    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
-    This is useful to prevent overriding any services that have a free allowance that is not the default.
-
-    If missing_services_only is false then add or update annual billing for all active services.
-    This is useful to ensure all services start the new year with the correct annual billing.
-    """
-    if missing_services_only:
-        active_services = Service.query.filter(
-            Service.active
-        ).outerjoin(
-            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
-        ).filter(
-            AnnualBilling.id == None  # noqa
-        ).all()
-    else:
-        active_services = Service.query.filter(
-            Service.active
-        ).all()
-    previous_year = year - 1
-    services_with_zero_free_allowance = db.session.query(AnnualBilling.service_id).filter(
-        AnnualBilling.financial_year_start == previous_year,
-        AnnualBilling.free_sms_fragment_limit == 0
-    ).all()
-
-    for service in active_services:
-
-        # If a service has free_sms_fragment_limit for the previous year
-        # set the free allowance for this year to 0 as well.
-        # Else use the default free allowance for the service.
-        if service.id in [x.service_id for x in services_with_zero_free_allowance]:
-            print(f'update service {service.id} to 0')
-            dao_create_or_update_annual_billing_for_year(
-                service_id=service.id,
-                free_sms_fragment_limit=0,
-                financial_year_start=year
-            )
-        else:
-            print(f'update service {service.id} with default')
-            set_default_free_allowance_for_service(service, year)
-
-
-@click.option('-u', '--user-id', required=True)
-@notify_command(name='local-dev-broadcast-permissions')
-def local_dev_broadcast_permissions(user_id):
-    if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
-    user = User.query.filter_by(id=user_id).one()
-
-    user_broadcast_services = Service.query.filter(
-        Service.permissions.any(permission='broadcast'),
-        Service.users.any(id=user_id)
-    )
-
-    for service in user_broadcast_services:
-        permission_list = [
-            Permission(service_id=service.id, user_id=user_id, permission=permission)
-            for permission in [
-                'reject_broadcasts', 'cancel_broadcasts',  # required to create / approve
-                'create_broadcasts', 'approve_broadcasts',  # minimum for testing
-                'manage_templates',  # unlikely but might be useful
-                'view_activity',  # normally added on invite / service creation
-            ]
-        ]
-
-        permission_dao.set_user_service_permission(
-            user, service, permission_list, _commit=True, replace=True
-        )
--- a/app/complaint/complaint_rest.py
+++ b/app/complaint/complaint_rest.py
@@ -3,10 +3,7 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request

 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import (
-    fetch_count_of_complaints,
-    fetch_paginated_complaints,
-)
+from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
 from app.errors import register_errors
 from app.schema_validation import validate
 from app.utils import pagination_links
--- a/app/complaint/complaint_schema.py
+++ b/app/complaint/complaint_schema.py
@@ -1,6 +1,6 @@

 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "complaint count request schema",
    "type": "object",
    "title": "Complaint count request",
--- a/app/config.py
+++ b/app/config.py
@@ -1,6 +1,6 @@
-import json
-import os
 from datetime import timedelta
+import os
+import json

 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -27,15 +27,12 @@ class QueueNames(object):
    PROCESS_FTP = 'process-ftp-tasks'
    CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
    CALLBACKS = 'service-callbacks'
-    CALLBACKS_RETRY = 'service-callbacks-retry'
    LETTERS = 'letter-tasks'
    SMS_CALLBACKS = 'sms-callbacks'
    ANTIVIRUS = 'antivirus-tasks'
    SANITISE_LETTERS = 'sanitise-letter-tasks'
    SAVE_API_EMAIL = 'save-api-email-tasks'
    SAVE_API_SMS = 'save-api-sms-tasks'
-    BROADCASTS = 'broadcast-tasks'
-    GOVUK_ALERTS = 'govuk-alerts'

    @staticmethod
    def all_queues():
@@ -52,90 +49,63 @@ class QueueNames(object):
            QueueNames.NOTIFY,
            QueueNames.CREATE_LETTERS_PDF,
            QueueNames.CALLBACKS,
-            QueueNames.CALLBACKS_RETRY,
            QueueNames.LETTERS,
            QueueNames.SMS_CALLBACKS,
            QueueNames.SAVE_API_EMAIL,
-            QueueNames.SAVE_API_SMS,
-            QueueNames.BROADCASTS,
+            QueueNames.SAVE_API_SMS
        ]


-class BroadcastProvider:
-    EE = 'ee'
-    VODAFONE = 'vodafone'
-    THREE = 'three'
-    O2 = 'o2'
-
-    PROVIDERS = [EE, VODAFONE, THREE, O2]
-
-
 class TaskNames(object):
    PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
    ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
    SCAN_FILE = 'scan-file'
    SANITISE_LETTER = 'sanitise-and-upload-letter'
    CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter'
-    PUBLISH_GOVUK_ALERTS = 'publish-govuk-alerts'
-    RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter'


 class Config(object):
    # URL of admin app
-    ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL')
+    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')

    # URL of api app (on AWS this is the internal api endpoint)
-    API_HOST_NAME = os.environ.get('API_HOST_NAME')
+    API_HOST_NAME = os.getenv('API_HOST_NAME')

    # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = 'notify-admin'
-    GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove?
-
-    INTERNAL_CLIENT_API_KEYS = json.loads(
-        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}')
-    ) # TODO: handled by varsfile?
+    API_INTERNAL_SECRETS = json.loads(os.environ.get('API_INTERNAL_SECRETS', '[]'))

    # encyption secret/salt
-    ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
-    SECRET_KEY = os.environ.get('SECRET_KEY')
-    DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT')
+    SECRET_KEY = os.getenv('SECRET_KEY')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')

    # DB conection string
-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
-
-    # AWS SMS
-    AWS_PINPOINT_REGION = os.environ.get("AWS_PINPOINT_REGION")
-    AWS_US_TOLL_FREE_NUMBER = os.environ.get("AWS_US_TOLL_FREE_NUMBER")
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')

    # MMG API Key
-    MMG_API_KEY = os.environ.get('MMG_API_KEY', 'placeholder')
+    MMG_API_KEY = os.getenv('MMG_API_KEY')

    # Firetext API Key
-    FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
-    FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
-    
-    # Whether to ignore POSTs from SNS for replies to SMS we sent
-    RECEIVE_INBOUND_SMS = False
+    FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")

-    # Use notify.sandbox.10x sending domain unless overwritten by environment
-    NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
-    
-    # AWS SNS topics for delivery receipts
-    VALIDATE_SNS_TOPICS = True
-    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound']
+    # Prefix to identify queues in SQS
+    NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')

    # URL of redis instance
-    REDIS_URL = os.environ.get('REDIS_URL')
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
+    REDIS_URL = os.getenv('REDIS_URL')
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
    EXPIRE_CACHE_TEN_MINUTES = 600
    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60

+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = False
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
+
    # Zendesk
    ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')

    # Logging
    DEBUG = False
-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH')
+    NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')

    # Cronitor
    CRONITOR_ENABLED = False
@@ -144,12 +114,16 @@ class Config(object):
    # Antivirus
    ANTIVIRUS_ENABLED = True

+    # Broadcast Messaging
+    CBC_PROXY_URL = None
+
    ###########################
    # Default config values ###
    ###########################

    NOTIFY_ENVIRONMENT = 'development'
-    AWS_REGION = 'us-west-2'
+    ADMIN_CLIENT_USER_NAME = 'notify-admin'
+    AWS_REGION = 'eu-west-1'
    INVITATION_EXPIRATION_DAYS = 2
    NOTIFY_APP_NAME = 'api'
    SQLALCHEMY_RECORD_QUERIES = False
@@ -162,10 +136,7 @@ class Config(object):
    API_PAGE_SIZE = 250
    TEST_MESSAGE_FILENAME = 'Test message'
    ONE_OFF_MESSAGE_FILENAME = 'Report'
-    MAX_VERIFY_CODE_COUNT = 5
-    MAX_FAILED_LOGIN_COUNT = 10
-
-    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
+    MAX_VERIFY_CODE_COUNT = 10

    # be careful increasing this size without being sure that we won't see slowness in pysftp
    MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
@@ -175,8 +146,8 @@ class Config(object):

    # these should always add up to 100%
    SMS_PROVIDER_RESTING_POINTS = {
-        'mmg': 50,
-        'firetext': 50
+        'mmg': 60,
+        'firetext': 40
    }

    NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
@@ -186,7 +157,7 @@ class Config(object):
    SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
    EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
    NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
-    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password
+    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
    ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
    CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
    SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
@@ -197,169 +168,152 @@ class Config(object):
    MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
    MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
    MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = '18446120782'
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
-    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
-    # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
+    MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'

-    CELERY = {
-        'broker_url': REDIS_URL,
-        'broker_transport_options': {
-            'visibility_timeout': 310,
-        },
-        'timezone': 'Europe/London',
-        'imports': [
-            'app.celery.tasks',
-            'app.celery.scheduled_tasks',
-            'app.celery.reporting_tasks',
-            'app.celery.nightly_tasks',
-        ],
-        # this is overriden by the -Q command, but locally, we should read from all queues
-        'task_queues': [
-            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
-        ],
-        'beat_schedule': {
-            # app/celery/scheduled_tasks.py
-            'run-scheduled-jobs': {
-                'task': 'run-scheduled-jobs',
-                'schedule': crontab(minute='0,15,30,45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-verify-codes': {
-                'task': 'delete-verify-codes',
-                'schedule': timedelta(minutes=63),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-invitations': {
-                'task': 'delete-invitations',
-                'schedule': timedelta(minutes=66),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'switch-current-sms-provider-on-slow-delivery': {
-                'task': 'switch-current-sms-provider-on-slow-delivery',
-                'schedule': crontab(),  # Every minute
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-job-status': {
-                'task': 'check-job-status',
-                'schedule': crontab(),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'tend-providers-back-to-middle': {
-                'task': 'tend-providers-back-to-middle',
-                'schedule': crontab(minute='*/5'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-missing-rows-in-completed-jobs': {
-                'task': 'check-for-missing-rows-in-completed-jobs',
-                'schedule': crontab(minute='*/10'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'replay-created-notifications': {
-                'task': 'replay-created-notifications',
-                'schedule': crontab(minute='0, 15, 30, 45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # app/celery/nightly_tasks.py
-            'timeout-sending-notifications': {
-                'task': 'timeout-sending-notifications',
-                'schedule': crontab(hour=0, minute=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'create-nightly-billing': {
-                'task': 'create-nightly-billing',
-                'schedule': crontab(hour=0, minute=15),
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'create-nightly-notification-status': {
-                'task': 'create-nightly-notification-status',
-                'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-notifications-older-than-retention': {
-                'task': 'delete-notifications-older-than-retention',
-                'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-inbound-sms': {
-                'task': 'delete-inbound-sms',
-                'schedule': crontab(hour=1, minute=40),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'save-daily-notification-processing-time': {
-                'task': 'save-daily-notification-processing-time',
-                'schedule': crontab(hour=2, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove_sms_email_jobs': {
-                'task': 'remove_sms_email_jobs',
-                'schedule': crontab(hour=4, minute=0),
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'remove_letter_jobs': {
-                'task': 'remove_letter_jobs',
-                'schedule': crontab(hour=4, minute=20),
-                # since we mark jobs as archived
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'check-if-letters-still-in-created': {
-                'task': 'check-if-letters-still-in-created',
-                'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-if-letters-still-pending-virus-check': {
-                'task': 'check-if-letters-still-pending-virus-check',
-                'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
-                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
-                'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-letter-notifications-still-sending': {
-                'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=17, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # The collate-letter-pdf does assume it is called in an hour that BST does not make a
-            # difference to the truncate date which translates to the filename to process
-            'collate-letter-pdfs-to-be-sent': {
-                'task': 'collate-letter-pdfs-to-be-sent',
-                'schedule': crontab(hour=17, minute=50),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-no-letter-ack-file': {
-                'task': 'raise-alert-if-no-letter-ack-file',
-                'schedule': crontab(hour=23, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'trigger-link-tests': {
-                'task': 'trigger-link-tests',
-                'schedule': timedelta(minutes=15),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'auto-expire-broadcast-messages': {
-                'task': 'auto-expire-broadcast-messages',
-                'schedule': timedelta(minutes=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove-yesterdays-planned-tests-on-govuk-alerts': {
-                'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
-                'schedule': crontab(hour=00, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-        }
+    BROKER_URL = 'sqs://'
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 310,
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
    }
-
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = 'Europe/London'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TASK_SERIALIZER = 'json'
+    # on reporting worker, restart workers after each task is executed to help prevent memory leaks
+    CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD')
    # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
-    if os.environ.get('CELERYD_PREFETCH_MULTIPLIER'):
-        CELERY['worker_prefetch_multiplier'] = os.environ.get('CELERYD_PREFETCH_MULTIPLIER')
+    if os.getenv('CELERYD_PREFETCH_MULTIPLIER'):
+        CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
+    CELERY_IMPORTS = (
+        'app.celery.tasks',
+        'app.celery.scheduled_tasks',
+        'app.celery.reporting_tasks',
+        'app.celery.nightly_tasks',
+    )
+    CELERYBEAT_SCHEDULE = {
+        # app/celery/scheduled_tasks.py
+        'run-scheduled-jobs': {
+            'task': 'run-scheduled-jobs',
+            'schedule': crontab(minute='0,15,30,45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-verify-codes': {
+            'task': 'delete-verify-codes',
+            'schedule': timedelta(minutes=63),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-invitations': {
+            'task': 'delete-invitations',
+            'schedule': timedelta(minutes=66),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-job-status': {
+            'task': 'check-job-status',
+            'schedule': crontab(),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'tend-providers-back-to-middle': {
+            'task': 'tend-providers-back-to-middle',
+            'schedule': crontab(minute='*/5'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-missing-rows-in-completed-jobs': {
+            'task': 'check-for-missing-rows-in-completed-jobs',
+            'schedule': crontab(minute='*/10'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'replay-created-notifications': {
+            'task': 'replay-created-notifications',
+            'schedule': crontab(minute='0, 15, 30, 45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # app/celery/nightly_tasks.py
+        'timeout-sending-notifications': {
+            'task': 'timeout-sending-notifications',
+            'schedule': crontab(hour=0, minute=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=0, minute=15),
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'create-nightly-notification-status': {
+            'task': 'create-nightly-notification-status',
+            'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'delete-notifications-older-than-retention': {
+            'task': 'delete-notifications-older-than-retention',
+            'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(hour=1, minute=40),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'remove_sms_email_jobs': {
+            'task': 'remove_sms_email_jobs',
+            'schedule': crontab(hour=4, minute=0),
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'remove_letter_jobs': {
+            'task': 'remove_letter_jobs',
+            'schedule': crontab(hour=4, minute=20),
+            # since we mark jobs as archived
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+            'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+            'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-letter-notifications-still-sending': {
+            'task': 'raise-alert-if-letter-notifications-still-sending',
+            'schedule': crontab(hour=15, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-to-be-sent': {
+            'task': 'collate-letter-pdfs-to-be-sent',
+            'schedule': crontab(hour=17, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-no-letter-ack-file': {
+            'task': 'raise-alert-if-no-letter-ack-file',
+            'schedule': crontab(hour=23, minute=00),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+    }
+    CELERY_QUEUES = []

    FROM_NUMBER = 'development'

-    STATSD_HOST = os.environ.get('STATSD_HOST')
+    STATSD_HOST = os.getenv('STATSD_HOST')
    STATSD_PORT = 8125
    STATSD_ENABLED = bool(STATSD_HOST)

@@ -379,11 +333,15 @@ class Config(object):
    FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
    MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
    MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
-    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
-    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
+    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
+    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')

    HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))

+    # Format is as follows:
+    # {"dataset_1": "token_1", ...}
+    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
+
    TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
    TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')

@@ -393,18 +351,15 @@ class Config(object):
    # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
    MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
    FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
+    SES_STUB_URL = os.environ.get("SES_STUB_URL")

-    AWS_REGION = 'us-west-2'
+    AWS_REGION = 'eu-west-1'

-    CBC_PROXY_ENABLED = True
+    # CBC Proxy
+    # if the access keys are empty then noop client is used
    CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
    CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '')

-    ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE}
-
-    # as defined in api db migration 0331_add_broadcast_org.py
-    BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
-

 ######################
 # Config overrides ###
@@ -414,30 +369,18 @@ class Development(Config):
    DEBUG = True
    SQLALCHEMY_ECHO = False

-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
+    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
+    CONTACT_LIST_BUCKET_NAME = 'development-contact-list'
+    TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'

-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2')
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2')
-    # TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
-    # DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
-    # LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
-    # LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
-    # INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
-    # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
-    # LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
-
-    # INTERNAL_CLIENT_API_KEYS = {
-    #     Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
-    #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
-    # }
-
-    SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
+    API_INTERNAL_SECRETS = ['dev-notify-secret-key']
+    SECRET_KEY = 'dev-notify-secret-key'
    DANGEROUS_SALT = 'dev-notify-salt'

    MMG_INBOUND_SMS_AUTH = ['testkey']
@@ -445,21 +388,23 @@ class Development(Config):

    NOTIFY_ENVIRONMENT = 'development'
    NOTIFY_LOG_PATH = 'application.log'
+    NOTIFICATION_QUEUE_PREFIX = 'development'
+    NOTIFY_EMAIL_DOMAIN = "notify.tools"

-    NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
+    SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
+    REDIS_URL = 'redis://localhost:6379/0'

-    SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
+    ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'

-    ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
-
-    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
-
-    API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011')
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )

+    API_HOST_NAME = "http://localhost:6011"
    API_RATE_LIMIT_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']

-    CBC_PROXY_ENABLED = False
+    CBC_PROXY_URL = 'http://localhost:8080'


 class Test(Development):
@@ -477,24 +422,26 @@ class Test(Development):

    CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
    CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
-    # TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
-    # DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
-    # LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
-    # LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
-    # INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
-    # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
-    # LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
+    TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'

-    # this is overriden in CI
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api')
+    # this is overriden in jenkins and on cloudfoundry
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')

-    CELERY = {
-        **Config.CELERY,
-        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
-    }
+    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'

    ANTIVIRUS_ENABLED = True

+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
    API_RATE_LIMIT_ENABLED = True
    API_HOST_NAME = "http://localhost:6011"

@@ -502,25 +449,25 @@ class Test(Development):
    FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
    TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'

+    CBC_PROXY_URL = 'http://test-cbc-proxy'
+
    MMG_URL = 'https://example.com/mmg'
    FIRETEXT_URL = 'https://example.com/firetext'

-    CBC_PROXY_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
-

 class Preview(Config):
    NOTIFY_EMAIL_DOMAIN = 'notify.works'
    NOTIFY_ENVIRONMENT = 'preview'
    CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
    CONTACT_LIST_BUCKET_NAME = 'preview-contact-list'
-    # TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
-    # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
-    # LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
-    # LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
-    # INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
-    # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
-    # LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
+    TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-preview.cloudapps.digital'
    FROM_NUMBER = 'preview'
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = False
@@ -531,49 +478,40 @@ class Staging(Config):
    NOTIFY_ENVIRONMENT = 'staging'
    CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
    CONTACT_LIST_BUCKET_NAME = 'staging-contact-list'
-    # TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
-    # DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
-    # LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
-    # LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
-    # INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
-    # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
-    # LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
+    TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-staging.cloudapps.digital'
    FROM_NUMBER = 'stage'
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = True


 class Live(Config):
+    NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
    NOTIFY_ENVIRONMENT = 'live'
-    # buckets
-    CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
-    CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
-    CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
-    CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
-    # TODO: verify below buckets only used for letters
-    # TEST_LETTERS_BUCKET_NAME = 'production-test-letters' # not created in gsa sandbox
-    # DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' # not created in gsa sandbox
-    # LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' # not created in gsa sandbox
-    # LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan' # not created in gsa sandbox
-    # INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf' # not created in gsa sandbox
-    # TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' # not created in gsa sandbox
-    # LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise' # not created in gsa sandbox
-
-    FROM_NUMBER = 'US Notify'
+    CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
+    CONTACT_LIST_BUCKET_NAME = 'production-contact-list'
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-production.cloudapps.digital'
+    FROM_NUMBER = 'GOVUK'
+    PERFORMANCE_PLATFORM_ENABLED = True
    API_RATE_LIMIT_ENABLED = True
    CHECK_PROXY_HEADER = True
    SES_STUB_URL = None
+
    CRONITOR_ENABLED = True

-    # DEBUG = True
-    REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
-
-    NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log')
-

 class CloudFoundryConfig(Config):
    pass
@@ -585,12 +523,12 @@ class Sandbox(CloudFoundryConfig):
    NOTIFY_ENVIRONMENT = 'sandbox'
    CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
    CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list'
-    # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
-    # TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
-    # DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
-    # LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
-    # LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
-    # INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
    FROM_NUMBER = 'sandbox'


--- a/app/cronitor.py
+++ b/app/cronitor.py
@@ -1,19 +1,15 @@
-from functools import wraps
-
 import requests
+from functools import wraps
 from flask import current_app


 def cronitor(task_name):
+    # check if task_name is in config
    def decorator(func):
        def ping_cronitor(command):
            if not current_app.config['CRONITOR_ENABLED']:
                return

-            # it's useful to have a log that a periodic task has started in case it
-            # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
-
            task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
            if not task_slug:
                current_app.logger.error(
@@ -42,11 +38,13 @@ def cronitor(task_name):
        @wraps(func)
        def inner_decorator(*args, **kwargs):
            ping_cronitor('run')
-            status = 'fail'
            try:
                ret = func(*args, **kwargs)
                status = 'complete'
                return ret
+            except Exception:
+                status = 'fail'
+                raise
            finally:
                ping_cronitor(status)

--- a/app/dao/init.py
+++ b/app/dao/init.py
@@ -1,5 +1,4 @@
 from sqlalchemy.exc import SQLAlchemyError
-
 from app import db


--- a/app/dao/annual_billing_dao.py
+++ b/app/dao/annual_billing_dao.py
@@ -1,12 +1,10 @@
-from flask import current_app
-
 from app import db
-from app.dao.dao_utils import autocommit
-from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.dao_utils import transactional
 from app.models import AnnualBilling
+from app.dao.date_util import get_current_financial_year_start_year


-@autocommit
+@transactional
 def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
    result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)

@@ -25,7 +23,7 @@ def dao_get_annual_billing(service_id):
    ).order_by(AnnualBilling.financial_year_start).all()


-@autocommit
+@transactional
 def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
    AnnualBilling.query.filter(
        AnnualBilling.service_id == service_id,
@@ -51,42 +49,3 @@ def dao_get_all_free_sms_fragment_limit(service_id):
    return AnnualBilling.query.filter_by(
        service_id=service_id,
    ).order_by(AnnualBilling.financial_year_start).all()
-
-
-def set_default_free_allowance_for_service(service, year_start=None):
-    default_free_sms_fragment_limits = {
-        'federal': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'state': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'other': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        }
-    }
-    if not year_start:
-        year_start = get_current_financial_year_start_year()
-    # handle cases where the year is less than 2020 or greater than 2021
-    if year_start < 2020:
-        year_start = 2020
-    if year_start > 2022:
-        year_start = 2022
-    if service.organisation_type:
-        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
-    else:
-        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
-                                f"{default_free_sms_fragment_limits['other'][year_start]}")
-        free_allowance = default_free_sms_fragment_limits['other'][year_start]
-
-    return dao_create_or_update_annual_billing_for_year(
-        service.id,
-        free_allowance,
-        year_start
-    )
--- a/app/dao/api_key_dao.py
+++ b/app/dao/api_key_dao.py
@@ -1,14 +1,18 @@
 import uuid
 from datetime import datetime, timedelta

-from sqlalchemy import func, or_
-
 from app import db
-from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey

+from app.dao.dao_utils import (
+    transactional,
+    version_class
+)

-@autocommit
+from sqlalchemy import or_, func
+
+
+@transactional
@version_class(ApiKey)
 def save_model_api_key(api_key):
    if not api_key.id:
@@ -17,7 +21,7 @@ def save_model_api_key(api_key):
    db.session.add(api_key)


-@autocommit
+@transactional
@version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
    api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
--- a/app/dao/broadcast_message_dao.py
+++ b/app/dao/broadcast_message_dao.py
@@ -1,20 +1,4 @@
-import uuid
-from datetime import datetime
-
-from sqlalchemy import desc
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    BroadcastEvent,
-    BroadcastMessage,
-    BroadcastProvider,
-    BroadcastProviderMessage,
-    BroadcastProviderMessageNumber,
-    BroadcastProviderMessageStatus,
-    BroadcastStatusType,
-    ServiceBroadcastSettings,
-)
+from app.models import BroadcastMessage, BroadcastEvent


 def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id):
@@ -24,19 +8,12 @@ def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service
    ).one()


-def dao_get_broadcast_message_by_references_and_service_id(references_to_original_broadcast, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.status.in_((
-            BroadcastStatusType.PENDING_APPROVAL,
-            BroadcastStatusType.BROADCASTING,
-        )),
-        BroadcastMessage.reference.in_(references_to_original_broadcast),
-        BroadcastMessage.service_id == service_id
-    ).one()
+def dao_get_broadcast_message_by_id(broadcast_message_id):
+    return BroadcastMessage.query.get(broadcast_message_id)


 def dao_get_broadcast_event_by_id(broadcast_event_id):
-    return BroadcastEvent.query.filter(BroadcastEvent.id == broadcast_event_id).one()
+    return BroadcastEvent.query.get(broadcast_event_id)


 def dao_get_broadcast_messages_for_service(service_id):
@@ -45,27 +22,6 @@ def dao_get_broadcast_messages_for_service(service_id):
    ).order_by(BroadcastMessage.created_at)


-def dao_get_all_broadcast_messages():
-    return db.session.query(
-        BroadcastMessage.id,
-        BroadcastMessage.reference,
-        ServiceBroadcastSettings.channel,
-        BroadcastMessage.content,
-        BroadcastMessage.areas,
-        BroadcastMessage.status,
-        BroadcastMessage.starts_at,
-        BroadcastMessage.finishes_at,
-        BroadcastMessage.approved_at,
-        BroadcastMessage.cancelled_at,
-    ).join(
-        ServiceBroadcastSettings, ServiceBroadcastSettings.service_id == BroadcastMessage.service_id
-    ).filter(
-        BroadcastMessage.starts_at >= datetime(2021, 5, 25, 0, 0, 0),
-        BroadcastMessage.stubbed == False,  # noqa
-        BroadcastMessage.status.in_(BroadcastStatusType.LIVE_STATUSES)
-    ).order_by(desc(BroadcastMessage.starts_at)).all()
-
-
 def get_earlier_events_for_broadcast_event(broadcast_event_id):
    """
    This is used to build up the references list.
@@ -78,28 +34,3 @@ def get_earlier_events_for_broadcast_event(broadcast_event_id):
    ).order_by(
        BroadcastEvent.sent_at.asc()
    ).all()
-
-
-@autocommit
-def create_broadcast_provider_message(broadcast_event, provider):
-    broadcast_provider_message_id = uuid.uuid4()
-    provider_message = BroadcastProviderMessage(
-        id=broadcast_provider_message_id,
-        broadcast_event=broadcast_event,
-        provider=provider,
-        status=BroadcastProviderMessageStatus.SENDING,
-    )
-    db.session.add(provider_message)
-    db.session.commit()
-    provider_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        provider_message_number = BroadcastProviderMessageNumber(
-            broadcast_provider_message_id=broadcast_provider_message_id)
-        db.session.add(provider_message_number)
-        db.session.commit()
-    return provider_message
-
-
-@autocommit
-def update_broadcast_provider_message_status(broadcast_provider_message, *, status):
-    broadcast_provider_message.status = status
--- a/app/dao/broadcast_service_dao.py
+++ b/app/dao/broadcast_service_dao.py
@@ -1,99 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    BROADCAST_TYPE,
-    EMAIL_AUTH_TYPE,
-    INVITE_PENDING,
-    VIEW_ACTIVITY,
-    ApiKey,
-    InvitedUser,
-    Organisation,
-    Permission,
-    Service,
-    ServiceBroadcastSettings,
-    ServicePermission,
-)
-
-
-@autocommit
-@version_class(Service)
-def set_broadcast_service_type(service, service_mode, broadcast_channel, provider_restriction):
-    insert_or_update_service_broadcast_settings(
-        service, channel=broadcast_channel, provider_restriction=provider_restriction
-    )
-
-    # Remove all permissions and add broadcast permission
-    if not service.has_permission(BROADCAST_TYPE):
-        service_permission = ServicePermission(service_id=service.id, permission=BROADCAST_TYPE)
-        db.session.add(service_permission)
-
-    ServicePermission.query.filter(
-        ServicePermission.service_id == service.id,
-        ServicePermission.permission != BROADCAST_TYPE,
-        # Email auth is an exception to the other service permissions (which relate to what type
-        # of notifications a service can send) where a broadcast service is allowed to have the
-        # email auth permission (but doesn't have to)
-        ServicePermission.permission != EMAIL_AUTH_TYPE
-    ).delete()
-
-    # Refresh the service object as it has references to the service permissions but we don't yet
-    # want to commit the permission changes incase all of this needs to rollback
-    db.session.refresh(service)
-
-    # Set service count as live false always
-    service.count_as_live = False
-
-    # Set service into training mode or live mode
-    if service_mode == "live":
-        if service.restricted:
-            # Only update the go live at timestamp if this if moving from training mode
-            # to live mode, not if it's moving from one type of live mode service to another
-            service.go_live_at = datetime.utcnow()
-        service.restricted = False
-    else:
-        service.restricted = True
-        service.go_live_at = None
-
-    # Remove all user permissions apart from view_activity for the service users and invited users
-    Permission.query.filter(
-        Permission.service_id == service.id,
-        Permission.permission != VIEW_ACTIVITY
-    ).delete()
-    InvitedUser.query.filter_by(
-        service_id=service.id,
-        status=INVITE_PENDING
-    ).update({'permissions': VIEW_ACTIVITY})
-
-    # Revoke any API keys to avoid a regular API key being used to send alerts
-    ApiKey.query.filter_by(
-        service_id=service.id,
-        expiry_date=None,
-    ).update({
-        ApiKey.expiry_date: datetime.utcnow()
-    })
-
-    # Add service to organisation
-    organisation = Organisation.query.filter_by(
-        id=current_app.config['BROADCAST_ORGANISATION_ID']
-    ).one()
-    service.organisation_id = organisation.id
-    service.organisation_type = organisation.organisation_type
-
-    db.session.add(service)
-
-
-def insert_or_update_service_broadcast_settings(service, channel, provider_restriction="all"):
-    if not service.service_broadcast_settings:
-        settings = ServiceBroadcastSettings()
-        settings.service = service
-        settings.channel = channel
-        settings.provider = provider_restriction
-        db.session.add(settings)
-    else:
-        service.service_broadcast_settings.channel = channel
-        service.service_broadcast_settings.provider = provider_restriction
-        db.session.add(service.service_broadcast_settings)
--- a/app/dao/complaint_dao.py
+++ b/app/dao/complaint_dao.py
@@ -4,12 +4,12 @@ from flask import current_app
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import Complaint
 from app.utils import get_london_midnight_in_utc


-@autocommit
+@transactional
 def save_complaint(complaint):
    db.session.add(complaint)

--- a/app/dao/daily_sorted_letter_dao.py
+++ b/app/dao/daily_sorted_letter_dao.py
@@ -3,7 +3,7 @@ from datetime import datetime
 from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import DailySortedLetter


@@ -13,7 +13,7 @@ def dao_get_daily_sorted_letter_by_billing_day(billing_day):
    ).first()


-@autocommit
+@transactional
 def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
    '''
    This uses the Postgres upsert to avoid race conditions when two threads try and insert
--- a/app/dao/dao_utils.py
+++ b/app/dao/dao_utils.py
@@ -1,20 +1,16 @@
 import itertools
-from contextlib import contextmanager
 from functools import wraps

 from app import db
 from app.history_meta import create_history


-def autocommit(func):
+def transactional(func):
    @wraps(func)
    def commit_or_rollback(*args, **kwargs):
        try:
            res = func(*args, **kwargs)
-
-            if not db.session.registry().transaction.nested:
-                db.session.commit()
-
+            db.session.commit()
            return res
        except Exception:
            db.session.rollback()
@@ -22,20 +18,6 @@ def autocommit(func):
    return commit_or_rollback


-@contextmanager
-def transaction():
-    try:
-        db.session.begin_nested()
-        yield
-        db.session.commit()
-
-        if not db.session.registry().transaction.nested:
-            db.session.commit()
-    except Exception:
-        db.session.rollback()
-        raise
-
-
 class VersionOptions():

    def __init__(self, model_class, history_class=None, must_write_history=True):
@@ -93,7 +75,7 @@ def dao_rollback():
    db.session.rollback()


-@autocommit
+@transactional
 def dao_save_object(obj):
    # add/update object in db
    db.session.add(obj)
--- a/app/dao/date_util.py
+++ b/app/dao/date_util.py
@@ -1,7 +1,7 @@
-from datetime import date, datetime, time, timedelta
+from datetime import datetime, timedelta, date, time

+from notifications_utils.timezones import convert_bst_to_utc
 import pytz
-from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst


 def get_months_for_financial_year(year):
@@ -22,15 +22,6 @@ def get_financial_year(year):
    return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)


-def get_financial_year_dates(year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    return (
-        convert_utc_to_bst(year_start_datetime).date(),
-        convert_utc_to_bst(year_end_datetime).date()
-    )
-
-
 def get_current_financial_year():
    now = datetime.utcnow()
    current_month = int(now.strftime('%-m'))
--- a/app/dao/email_branding_dao.py
+++ b/app/dao/email_branding_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import EmailBranding


@@ -15,12 +15,12 @@ def dao_get_email_branding_by_name(email_branding_name):
    return EmailBranding.query.filter_by(name=email_branding_name).first()


-@autocommit
+@transactional
 def dao_create_email_branding(email_branding):
    db.session.add(email_branding)


-@autocommit
+@transactional
 def dao_update_email_branding(email_branding, **kwargs):
    for key, value in kwargs.items():
        setattr(email_branding, key, value or None)
--- a/app/dao/fact_billing_dao.py
+++ b/app/dao/fact_billing_dao.py
--- a/app/dao/fact_notification_status_dao.py
+++ b/app/dao/fact_notification_status_dao.py
@@ -1,89 +1,108 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time

-from sqlalchemy import Date, case, func
+from flask import current_app
+from notifications_utils.timezones import convert_bst_to_utc
+from sqlalchemy import case, func, Date
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import extract, literal
+from sqlalchemy.sql.expression import literal, extract
 from sqlalchemy.types import DateTime, Integer

 from app import db
-from app.dao.dao_utils import autocommit
 from app.models import (
-    KEY_TYPE_NORMAL,
-    KEY_TYPE_TEAM,
+    FactNotificationStatus,
    KEY_TYPE_TEST,
+    Notification,
    NOTIFICATION_CANCELLED,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_FAILED,
    NOTIFICATION_PENDING,
-    NOTIFICATION_PERMANENT_FAILURE,
    NOTIFICATION_SENDING,
    NOTIFICATION_SENT,
    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
-    FactNotificationStatus,
-    Notification,
-    NotificationAllTimeView,
+    NOTIFICATION_PERMANENT_FAILURE,
    Service,
    Template,
 )
+from app.dao.dao_utils import transactional
 from app.utils import (
    get_london_midnight_in_utc,
-    get_london_month_from_utc_column,
    midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
 )


-@autocommit
-def update_fact_notification_status(process_day, notification_type, service_id):
-    start_date = get_london_midnight_in_utc(process_day)
-    end_date = get_london_midnight_in_utc(process_day + timedelta(days=1))
+def fetch_notification_status_for_day(process_day, notification_type):
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))

-    # delete any existing rows in case some no longer exist e.g. if all messages are sent
-    FactNotificationStatus.query.filter(
-        FactNotificationStatus.bst_date == process_day,
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.service_id == service_id,
-    ).delete()
+    current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))

+    all_data_for_process_day = []
+    services = Service.query.all()
+    # for each service query notifications or notification_history for the day, depending on their data retention
+    for service in services:
+        table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
+
+        data_for_service_and_type = query_for_fact_status_data(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+            notification_type=notification_type,
+            service_id=service.id
+        )
+
+        all_data_for_process_day += data_for_service_and_type
+
+    return all_data_for_process_day
+
+
+def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
    query = db.session.query(
-        literal(process_day).label("process_day"),
-        NotificationAllTimeView.template_id,
-        literal(service_id).label("service_id"),
-        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
-        literal(notification_type).label("notification_type"),
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status,
+        table.template_id,
+        table.service_id,
+        func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        table.key_type,
+        table.status,
        func.count().label('notification_count')
    ).filter(
-        NotificationAllTimeView.created_at >= start_date,
-        NotificationAllTimeView.created_at < end_date,
-        NotificationAllTimeView.notification_type == notification_type,
-        NotificationAllTimeView.service_id == service_id,
-        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        table.created_at >= start_date,
+        table.created_at < end_date,
+        table.notification_type == notification_type,
+        table.service_id == service_id,
+        table.key_type != KEY_TYPE_TEST
    ).group_by(
-        NotificationAllTimeView.template_id,
-        NotificationAllTimeView.template_id,
+        table.template_id,
+        table.service_id,
        'job_id',
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status
+        table.key_type,
+        table.status
    )
+    return query.all()

-    db.session.connection().execute(
-        insert(FactNotificationStatus.__table__).from_select(
-            [
-                FactNotificationStatus.bst_date,
-                FactNotificationStatus.template_id,
-                FactNotificationStatus.service_id,
-                FactNotificationStatus.job_id,
-                FactNotificationStatus.notification_type,
-                FactNotificationStatus.key_type,
-                FactNotificationStatus.notification_status,
-                FactNotificationStatus.notification_count
-            ],
-            query
+
+@transactional
+def update_fact_notification_status(data, process_day, notification_type):
+    table = FactNotificationStatus.__table__
+    FactNotificationStatus.query.filter(
+        FactNotificationStatus.bst_date == process_day,
+        FactNotificationStatus.notification_type == notification_type
+    ).delete()
+
+    for row in data:
+        stmt = insert(table).values(
+            bst_date=process_day,
+            template_id=row.template_id,
+            service_id=row.service_id,
+            job_id=row.job_id,
+            notification_type=notification_type,
+            key_type=row.key_type,
+            notification_status=row.status,
+            notification_count=row.notification_count,
        )
-    )
+        db.session.connection().execute(stmt)


 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
@@ -411,37 +430,16 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
    return query.all()


-def get_total_notifications_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('emails'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('sms'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('letters'),
+def get_total_sent_notifications_for_day_and_type(day, notification_type):
+    result = db.session.query(
+        func.sum(FactNotificationStatus.notification_count).label('count')
    ).filter(
+        FactNotificationStatus.notification_type == notification_type,
        FactNotificationStatus.key_type != KEY_TYPE_TEST,
-    ).group_by(
-        FactNotificationStatus.bst_date
-    ).order_by(
-        FactNotificationStatus.bst_date
-    )
-    if start_date and end_date:
-        query = query.filter(
-            FactNotificationStatus.bst_date >= start_date,
-            FactNotificationStatus.bst_date <= end_date
-        )
-    return query.all()
+        FactNotificationStatus.bst_date == day,
+    ).scalar()
+
+    return result or 0


 def fetch_monthly_notification_statuses_per_service(start_date, end_date):
--- a/app/dao/fact_processing_time_dao.py
+++ b/app/dao/fact_processing_time_dao.py
@@ -1,53 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import case
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import FactProcessingTime
-
-
-@autocommit
-def insert_update_processing_time(processing_time):
-    '''
-    This uses the Postgres upsert to avoid race conditions when two threads try and insert
-    at the same row. The excluded object refers to values that we tried to insert but were
-    rejected.
-    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    '''
-    table = FactProcessingTime.__table__
-    stmt = insert(table).values(
-        bst_date=processing_time.bst_date,
-        messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs
-    )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.bst_date],
-        set_={
-            'messages_total': stmt.excluded.messages_total,
-            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
-            'updated_at': datetime.utcnow()
-        }
-    )
-    db.session.connection().execute(stmt)
-
-
-def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactProcessingTime.bst_date.cast(db.Text).label("date"),
-        FactProcessingTime.messages_total,
-        FactProcessingTime.messages_within_10_secs,
-        case([
-            (
-                FactProcessingTime.messages_total > 0,
-                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
-            ),
-            (FactProcessingTime.messages_total == 0, 100.0)
-        ]).label("percentage")
-    ).filter(
-        FactProcessingTime.bst_date >= start_date,
-        FactProcessingTime.bst_date <= end_date
-    ).order_by(FactProcessingTime.bst_date)
-
-    return query.all()
--- a/app/dao/inbound_numbers_dao.py
+++ b/app/dao/inbound_numbers_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import InboundNumber


@@ -19,13 +19,13 @@ def dao_get_inbound_number(inbound_number_id):
    return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()


-@autocommit
+@transactional
 def dao_set_inbound_number_to_service(service_id, inbound_number):
    inbound_number.service_id = service_id
    db.session.add(inbound_number)


-@autocommit
+@transactional
 def dao_set_inbound_number_active_flag(service_id, active):
    inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
    inbound_number.active = active
@@ -33,7 +33,7 @@ def dao_set_inbound_number_active_flag(service_id, active):
    db.session.add(inbound_number)


-@autocommit
+@transactional
 def dao_allocate_number_for_service(service_id, inbound_number_id):
    updated = InboundNumber.query.filter_by(
        id=inbound_number_id,
--- a/app/dao/inbound_sms_dao.py
+++ b/app/dao/inbound_sms_dao.py
@@ -1,21 +1,15 @@
 from flask import current_app
-from sqlalchemy import and_, desc
-from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import desc, and_
 from sqlalchemy.orm import aliased
+from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    InboundSms,
-    InboundSmsHistory,
-    Service,
-    ServiceDataRetention,
-)
+from app.dao.dao_utils import transactional
+from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
 from app.utils import midnight_n_days_ago


-@autocommit
+@transactional
 def dao_create_inbound_sms(inbound_sms):
    db.session.add(inbound_sms)

@@ -71,13 +65,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days):
 def _insert_inbound_sms_history(subquery, query_limit=10000):
    offset = 0
    inbound_sms_query = db.session.query(
-        InboundSms.id,
-        InboundSms.created_at,
-        InboundSms.service_id,
-        InboundSms.notify_number,
-        InboundSms.provider_date,
-        InboundSms.provider_reference,
-        InboundSms.provider
+        *[x.name for x in InboundSmsHistory.__table__.c]
    ).filter(InboundSms.id.in_(subquery))
    inbound_sms_count = inbound_sms_query.count()

@@ -119,7 +107,7 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
    return deleted


-@autocommit
+@transactional
 def delete_inbound_sms_older_than_retention():
    current_app.logger.info('Deleting inbound sms for services with flexible data retention')

--- a/app/dao/invited_org_user_dao.py
+++ b/app/dao/invited_org_user_dao.py
@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedOrganisationUser


--- a/app/dao/invited_user_dao.py
+++ b/app/dao/invited_user_dao.py
@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedUser


@@ -9,7 +9,7 @@ def save_invited_user(invited_user):
    db.session.commit()


-def get_invited_user_by_service_and_id(service_id, invited_user_id):
+def get_invited_user(service_id, invited_user_id):
    return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()


--- a/app/dao/jobs_dao.py
+++ b/app/dao/jobs_dao.py
@@ -2,30 +2,33 @@ import uuid
 from datetime import datetime, timedelta

 from flask import current_app
-from notifications_utils.letter_timings import (
-    CANCELLABLE_JOB_LETTER_STATUSES,
-    letter_can_be_cancelled,
+from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
+from sqlalchemy import (
+    asc,
+    desc,
+    func,
+    and_
 )
-from sqlalchemy import and_, asc, desc, func

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.dao.templates_dao import dao_get_template_by_id
+from app.utils import midnight_n_days_ago
+
 from app.models import (
-    JOB_STATUS_CANCELLED,
+    Job,
    JOB_STATUS_FINISHED,
    JOB_STATUS_PENDING,
    JOB_STATUS_SCHEDULED,
    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    NOTIFICATION_CREATED,
-    FactNotificationStatus,
-    Job,
    Notification,
-    ServiceDataRetention,
    Template,
+    ServiceDataRetention,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_CANCELLED,
+    JOB_STATUS_CANCELLED,
+    FactNotificationStatus
 )
-from app.utils import midnight_n_days_ago


 def dao_get_notification_outcomes_for_job(service_id, job_id):
@@ -183,7 +186,7 @@ def dao_get_jobs_older_than_data_retention(notification_types):
    return jobs


-@autocommit
+@transactional
 def dao_cancel_letter_job(job):
    number_of_notifications_cancelled = Notification.query.filter(
        Notification.job_id == job.id
--- a/app/dao/letter_branding_dao.py
+++ b/app/dao/letter_branding_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import LetterBranding


@@ -15,12 +15,12 @@ def dao_get_all_letter_branding():
    return LetterBranding.query.order_by(LetterBranding.name).all()


-@autocommit
+@transactional
 def dao_create_letter_branding(letter_branding):
    db.session.add(letter_branding)


-@autocommit
+@transactional
 def dao_update_letter_branding(letter_branding_id, **kwargs):
    letter_branding = LetterBranding.query.get(letter_branding_id)
    for key, value in kwargs.items():
--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -1,53 +1,56 @@
-from datetime import datetime, timedelta
+import functools
 from itertools import groupby
 from operator import attrgetter
+from datetime import (
+    datetime,
+    timedelta,
+)

 from botocore.exceptions import ClientError
 from flask import current_app
-from notifications_utils.international_billing_rates import (
-    INTERNATIONAL_BILLING_RATES,
-)
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
    validate_and_format_email_address,
+    InvalidEmailError,
+    try_validate_and_format_phone_number
 )
 from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
-from sqlalchemy import and_, asc, desc, func, or_, union
+from sqlalchemy import (desc, func, asc, and_, or_)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
 from werkzeug.datastructures import MultiDict

-from app import create_uuid, db, statsd_client
-from app.dao.dao_utils import autocommit
-from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
+from app import db, create_uuid
+from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
+from app.dao.dao_utils import transactional
+from app.letters.utils import get_letter_pdf_filename
 from app.models import (
-    EMAIL_TYPE,
+    FactNotificationStatus,
+    Notification,
+    NotificationHistory,
+    ProviderDetails,
    KEY_TYPE_NORMAL,
    KEY_TYPE_TEST,
    LETTER_TYPE,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
    NOTIFICATION_PENDING,
    NOTIFICATION_PENDING_VIRUS_CHECK,
-    NOTIFICATION_PERMANENT_FAILURE,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
+    NOTIFICATION_TECHNICAL_FAILURE,
    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENT,
    SMS_TYPE,
-    FactNotificationStatus,
-    Notification,
-    NotificationHistory,
-    ProviderDetails,
-)
-from app.utils import (
-    escape_special_characters,
-    get_london_midnight_in_utc,
-    midnight_n_days_ago,
+    EMAIL_TYPE,
+    ServiceDataRetention,
+    Service,
 )
+from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters
+from app.clients.sms.firetext import get_message_status_and_reason_from_firetext_code


 def dao_get_last_date_template_was_used(template_id, service_id):
@@ -72,7 +75,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
    return last_date


-@autocommit
+@transactional
 def dao_create_notification(notification):
    if not notification.id:
        # need to populate defaulted fields before we create the notification history object
@@ -83,26 +86,40 @@ def dao_create_notification(notification):
    db.session.add(notification)


-def country_records_delivery(phone_prefix):
-    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
-    return dlr and dlr.lower() == 'yes'
-
-def _decide_permanent_temporary_failure(current_status, status):
-    # If we go from pending to delivered we need to set failure type as temporary-failure
-    if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+def _decide_permanent_temporary_failure(status, notification, detailed_status_code=None):
+    # If we get failure status from Firetext, we want to know if this is temporary or permanent failure.
+    # So we check the failure code to learn that.
+    # If there is no failure code, or we do not recognise the failure code, we do the following:
+    # if notifitcation goes form status pending to status failure, we mark it as temporary failure;
+    # if notification goes straight to status failure, we mark it as permanent failure.
+    if status == NOTIFICATION_PERMANENT_FAILURE and detailed_status_code not in [None, '000']:
+        try:
+            status, reason = get_message_status_and_reason_from_firetext_code(detailed_status_code)
+            current_app.logger.info(f'Updating notification id {notification.id} to status {status}, reason: {reason}')
+            return status
+        except KeyError:
+            current_app.logger.warning(f'Failure code {detailed_status_code} from Firetext not recognised')
+    # fallback option:
+    if notification.status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
        status = NOTIFICATION_TEMPORARY_FAILURE
    return status


-def _update_notification_status(notification, status, provider_response=None):
-    status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
+def country_records_delivery(phone_prefix):
+    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
+    return dlr and dlr.lower() == 'yes'
+
+
+def _update_notification_status(notification, status, detailed_status_code=None):
+    status = _decide_permanent_temporary_failure(
+        status=status, notification=notification, detailed_status_code=detailed_status_code
+    )
    notification.status = status
-    if provider_response:
-        notification.provider_response = provider_response
    dao_update_notification(notification)
    return notification

-@autocommit
+
+@transactional
 def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
    notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()

@@ -138,7 +155,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
    )


-@autocommit
+@transactional
 def update_notification_status_by_reference(reference, status):
    # this is used to update letters and emails
    notification = Notification.query.filter(Notification.reference == reference).first()
@@ -160,12 +177,16 @@ def update_notification_status_by_reference(reference, status):
    )


-@autocommit
+@transactional
 def dao_update_notification(notification):
    notification.updated_at = datetime.utcnow()
    db.session.add(notification)


+def get_notification_for_job(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
    if page_size is None:
        page_size = current_app.config['PAGE_SIZE']
@@ -200,6 +221,10 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
    return query.one() if _raise else query.first()


+def get_notifications(filter_dict=None):
+    return _filter_query(Notification.query, filter_dict=filter_dict)
+
+
 def get_notifications_for_service(
        service_id,
        filter_dict=None,
@@ -213,8 +238,7 @@ def get_notifications_for_service(
        include_from_test_key=False,
        older_than=None,
        client_reference=None,
-        include_one_off=True,
-        error_out=True
+        include_one_off=True
 ):
    if page_size is None:
        page_size = current_app.config['PAGE_SIZE']
@@ -253,8 +277,7 @@ def get_notifications_for_service(
    return query.order_by(desc(Notification.created_at)).paginate(
        page=page,
        per_page=page_size,
-        count=count_pages,
-        error_out=error_out,
+        count=count_pages
    )


@@ -278,42 +301,51 @@ def _filter_query(query, filter_dict=None):
    return query


-@autocommit
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=50000):
+    current_app.logger.info(
+        'Deleting {} notifications for services with flexible data retention'.format(notification_type))
+
+    flexible_data_retention = ServiceDataRetention.query.filter(
+        ServiceDataRetention.notification_type == notification_type
+    ).all()
+    deleted = 0
+    for f in flexible_data_retention:
+        current_app.logger.info(
+            "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
+
+        day_to_delete_backwards_from = get_london_midnight_in_utc(
+            convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)
+
+        deleted += _move_notifications_to_notification_history(
+            notification_type, f.service_id, day_to_delete_backwards_from, qry_limit)
+
+    current_app.logger.info(
+        'Deleting {} notifications for services without flexible data retention'.format(notification_type))
+
+    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
+    services_with_data_retention = [x.service_id for x in flexible_data_retention]
+    service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
+
+    for service_id in service_ids_to_purge:
+        deleted += _move_notifications_to_notification_history(
+            notification_type, service_id, seven_days_ago, qry_limit)
+
+    current_app.logger.info('Finished deleting {} notifications'.format(notification_type))
+
+    return deleted
+
+
+@transactional
 def insert_notification_history_delete_notifications(
    notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
 ):
-    """
-    Delete up to 50,000 notifications that are past retention for a notification type and service.
-
-
-    Steps are as follows:
-
-    Create a temporary notifications table
-    Populate that table with up to 50k notifications that are to be deleted. (Note: no specified order)
-    Insert everything in the temp table into notification history
-    Delete from notifications if notification id is in the temp table
-    Drop the temp table (automatically when the transaction commits)
-
-    Temporary tables are in a separate postgres schema, and only visible to the current session (db connection,
-    in a celery task there's one connection per thread.)
-    """
    # Setting default query limit to 50,000 which take about 48 seconds on current table size
    # 10, 000 took 11s and 100,000 took 1 min 30 seconds.
-    select_into_temp_table = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, postage, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
+    drop_table_if_exists = """
+        DROP TABLE if exists NOTIFICATION_ARCHIVE
        """
-    select_into_temp_table_for_letters = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
+    select_into_temp_table = """
+         CREATE TEMP TABLE NOTIFICATION_ARCHIVE AS
         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
             client_reference, international, phone_prefix, rate_multiplier, notification_status,
@@ -322,7 +354,6 @@ def insert_notification_history_delete_notifications(
        WHERE service_id = :service_id
          AND notification_type = :notification_type
          AND created_at < :timestamp_to_delete_backwards_from
-          AND notification_status NOT IN ('pending-virus-check', 'created', 'sending')
          AND key_type in ('normal', 'team')
        limit :qry_limit
        """
@@ -344,35 +375,31 @@ def insert_notification_history_delete_notifications(
        "qry_limit": qry_limit
    }

-    select_to_use = select_into_temp_table_for_letters if notification_type == 'letter' else select_into_temp_table
-    db.session.execute(select_to_use, input_params)
+    db.session.execute(drop_table_if_exists)
+    db.session.execute(select_into_temp_table, input_params)

-    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
+    result = db.session.execute("select * from NOTIFICATION_ARCHIVE")

    db.session.execute(insert_query)

    db.session.execute(delete_query)

-    return result
+    db.session.execute("DROP TABLE NOTIFICATION_ARCHIVE")
+    return result.rowcount


-def move_notifications_to_notification_history(
-    notification_type,
-    service_id,
-    timestamp_to_delete_backwards_from,
-    qry_limit=50000
-):
+def _move_notifications_to_notification_history(notification_type, service_id, day_to_delete_backwards_from, qry_limit):
    deleted = 0
    if notification_type == LETTER_TYPE:
        _delete_letters_from_s3(
-            notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit
+            notification_type, service_id, day_to_delete_backwards_from, qry_limit
        )
    delete_count_per_call = 1
    while delete_count_per_call > 0:
        delete_count_per_call = insert_notification_history_delete_notifications(
            notification_type=notification_type,
            service_id=service_id,
-            timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
+            timestamp_to_delete_backwards_from=day_to_delete_backwards_from,
            qry_limit=qry_limit
        )
        deleted += delete_count_per_call
@@ -381,7 +408,7 @@ def move_notifications_to_notification_history(
    Notification.query.filter(
        Notification.notification_type == notification_type,
        Notification.service_id == service_id,
-        Notification.created_at < timestamp_to_delete_backwards_from,
+        Notification.created_at < day_to_delete_backwards_from,
        Notification.key_type == KEY_TYPE_TEST
    ).delete(synchronize_session=False)
    db.session.commit()
@@ -397,55 +424,77 @@ def _delete_letters_from_s3(
    ).filter(
        Notification.notification_type == notification_type,
        Notification.created_at < date_to_delete_from,
-        Notification.service_id == service_id,
-        # although letters in non completed statuses do have PDFs in s3, they do not exist in the
-        # production-letters-pdf bucket as they never made it that far so we do not try and delete
-        # them from it
-        Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)
+        Notification.service_id == service_id
    ).limit(query_limit).all()
    for letter in letters_to_delete_from_s3:
-        try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
-            letter_pdf.delete()
-        except ClientError:
-            current_app.logger.exception(
-                "Error deleting S3 object for letter: {}".format(letter.id))
-        except LetterPDFNotFound:
-            current_app.logger.warning(
-                "No S3 object to delete for letter: {}".format(letter.id))
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+        # I don't think we need this anymore, we should update the query to get letters sent 7 days ago
+        if letter.sent_at:
+            prefix = get_letter_pdf_filename(reference=letter.reference,
+                                             crown=letter.service.crown,
+                                             created_at=letter.created_at,
+                                             ignore_folder=letter.key_type == KEY_TYPE_TEST,
+                                             postage=letter.postage)
+            s3_objects = get_s3_bucket_objects(bucket_name=bucket_name, subfolder=prefix)
+            for s3_object in s3_objects:
+                try:
+                    remove_s3_object(bucket_name, s3_object['Key'])
+                except ClientError:
+                    current_app.logger.exception(
+                        "Could not delete S3 object with filename: {}".format(s3_object['Key']))


-@autocommit
+@transactional
 def dao_delete_notifications_by_id(notification_id):
    db.session.query(Notification).filter(
        Notification.id == notification_id
    ).delete(synchronize_session='fetch')


-def dao_timeout_notifications(cutoff_time, limit=100000):
-    """
-    Set email and SMS notifications (only) to "temporary-failure" status
-    if they're still sending from before the specified cutoff_time.
-    """
-    updated_at = datetime.utcnow()
-    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
-    new_status = NOTIFICATION_TEMPORARY_FAILURE
-
+def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
    notifications = Notification.query.filter(
-        Notification.created_at < cutoff_time,
+        Notification.created_at < timeout_start,
        Notification.status.in_(current_statuses),
-        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
-    ).limit(limit).all()
-
+        Notification.notification_type != LETTER_TYPE
+    ).all()
    Notification.query.filter(
-        Notification.id.in_([n.id for n in notifications]),
+        Notification.created_at < timeout_start,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type != LETTER_TYPE
    ).update(
        {'status': new_status, 'updated_at': updated_at},
        synchronize_session=False
    )
+    return notifications
+
+
+def dao_timeout_notifications(timeout_period_in_seconds):
+    """
+    Timeout SMS and email notifications by the following rules:
+
+    we never sent the notification to the provider for some reason
+        created -> technical-failure
+
+    the notification was sent to the provider but there was not a delivery receipt
+        sending -> temporary-failure
+        pending -> temporary-failure
+
+    Letter notifications are not timed out
+    """
+    timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
+    updated_at = datetime.utcnow()
+    timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
+
+    # Notifications still in created status are marked with a technical-failure:
+    technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
+
+    # Notifications still in sending or pending status are marked with a temporary-failure:
+    temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
+                                              NOTIFICATION_TEMPORARY_FAILURE)

    db.session.commit()
-    return notifications
+
+    return technical_failure_notifications, temporary_failure_notifications


 def is_delivery_slow_for_providers(
@@ -474,7 +523,6 @@ def is_delivery_slow_for_providers(
        ProviderDetails
    ).outerjoin(
        Notification, and_(
-            Notification.notification_type == SMS_TYPE,
            Notification.sent_by == ProviderDetails.identifier,
            Notification.created_at >= created_at,
            Notification.sent_at.isnot(None),
@@ -498,12 +546,15 @@ def is_delivery_slow_for_providers(
        slow_notifications = sum(row.count for row in rows if row.slow)

        slow_providers[provider] = (slow_notifications / total_notifications >= threshold)
-        statsd_client.gauge(f'slow-delivery.{provider}.ratio', slow_notifications / total_notifications)
+
+        current_app.logger.info("Slow delivery notifications count for provider {}: {} out of {}. Ratio {}".format(
+            provider, slow_notifications, total_notifications, slow_notifications / total_notifications
+        ))

    return slow_providers


-@autocommit
+@transactional
 def dao_update_notifications_by_reference(references, update_dict):
    updated_count = Notification.query.filter(
        Notification.reference.in_(references)
@@ -531,7 +582,6 @@ def dao_get_notifications_by_recipient_or_reference(
    statuses=None,
    page=1,
    page_size=None,
-    error_out=True,
 ):

    if notification_type == SMS_TYPE:
@@ -582,7 +632,7 @@ def dao_get_notifications_by_recipient_or_reference(
    results = db.session.query(Notification)\
        .filter(*filters)\
        .order_by(desc(Notification.created_at))\
-        .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
+        .paginate(page=page, per_page=page_size)
    return results


@@ -592,24 +642,31 @@ def dao_get_notification_by_reference(reference):
    ).one()


-def dao_get_notification_history_by_reference(reference):
+def dao_get_notification_or_history_by_reference(reference):
    try:
        # This try except is necessary because in test keys and research mode does not create notification history.
        # Otherwise we could just search for the NotificationHistory object
-        return Notification.query.filter(Notification.reference == reference).one()
+        return Notification.query.filter(
+            Notification.reference == reference
+        ).one()
    except NoResultFound:
-        return NotificationHistory.query.filter(NotificationHistory.reference == reference).one()
+        return NotificationHistory.query.filter(
+            NotificationHistory.reference == reference
+        ).one()


-def dao_get_notifications_processing_time_stats(start_date, end_date):
+def dao_get_notifications_by_references(references):
+    return Notification.query.filter(
+        Notification.reference.in_(references)
+    ).all()
+
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
    """
-    For a given time range, returns the number of notifications sent and the number of
-    those notifications that we processed within 10 seconds
-
    SELECT
-    count(notifications),
+    count(notification_history),
    coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
    WHERE
    created_at > 'START DATE' AND
    created_at < 'END DATE' AND
@@ -660,51 +717,20 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
    return notifications


-def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
+def dao_get_letters_to_be_printed(print_run_deadline, postage):
    """
-    Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
-    to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
-    get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese  start sending off
-    tasks to the notify-ftp app to send them.
-
-    CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
-    there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
-    and we should avoid modifying state of returned objects.
-
-    For more reading:
-    https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
-    https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
+    Return all letters created before the print run deadline that have not yet been sent
    """
    notifications = Notification.query.filter(
        Notification.created_at < convert_bst_to_utc(print_run_deadline),
        Notification.notification_type == LETTER_TYPE,
        Notification.status == NOTIFICATION_CREATED,
        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.postage == postage,
-        Notification.billable_units > 0
+        Notification.postage == postage
    ).order_by(
        Notification.service_id,
        Notification.created_at
-    ).yield_per(query_limit)
-    return notifications
-
-
-def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
-    notifications = db.session.query(
-        func.count(Notification.id).label('letters_count'),
-        func.sum(Notification.billable_units).label('sheets_count'),
-        Notification.postage
-    ).filter(
-        Notification.created_at < convert_bst_to_utc(print_run_deadline),
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.billable_units > 0
-    ).group_by(
-        Notification.postage
-    ).order_by(
-        Notification.postage
-    ).all()
+    ).limit(50000)
    return notifications


@@ -754,58 +780,14 @@ def dao_precompiled_letters_still_pending_virus_check():
 def _duplicate_update_warning(notification, status):
    current_app.logger.info(
        (
-            'Duplicate callback received for service {service_id}. '
-            'Notification ID {id} with type {type} sent by {sent_by}. '
-            'New status was {new_status}, current status is {old_status}. '
-            'This happened {time_diff} after being first set.'
+            'Duplicate callback received. Notification id {id} received a status update to {new_status}'
+            '{time_diff} after being set to {old_status}. {type} sent by {sent_by}'
        ).format(
            id=notification.id,
            old_status=notification.status,
            new_status=status,
            time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
            type=notification.notification_type,
-            sent_by=notification.sent_by,
-            service_id=notification.service_id
+            sent_by=notification.sent_by
        )
    )
-
-
-def get_service_ids_with_notifications_before(notification_type, timestamp):
-    return {
-        row.service_id
-        for row in db.session.query(
-            Notification.service_id
-        ).filter(
-            Notification.notification_type == notification_type,
-            Notification.created_at < timestamp
-        ).distinct()
-    }
-
-
-def get_service_ids_with_notifications_on_date(notification_type, date):
-    start_date = get_london_midnight_in_utc(date)
-    end_date = get_london_midnight_in_utc(date + timedelta(days=1))
-
-    notification_table_query = db.session.query(
-        Notification.service_id.label('service_id')
-    ).filter(
-        Notification.notification_type == notification_type,
-        # using >= + < is much more efficient than date(created_at)
-        Notification.created_at >= start_date,
-        Notification.created_at < end_date,
-    )
-
-    # Looking at this table is more efficient for historical notifications,
-    # provided the task to populate it has run before they were archived.
-    ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label('service_id')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.bst_date == date,
-    )
-
-    return {
-        row.service_id for row in db.session.query(union(
-            notification_table_query, ft_status_table_query
-        ).subquery()).distinct()
-    }
--- a/app/dao/organisation_dao.py
+++ b/app/dao/organisation_dao.py
@@ -1,8 +1,14 @@
 from sqlalchemy.sql.expression import func

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.models import Domain, Organisation, Service, User
+from app.dao.dao_utils import VersionOptions, transactional, version_class
+from app.models import (
+    Organisation,
+    Domain,
+    InvitedOrganisationUser,
+    Service,
+    User
+)


 def dao_get_organisations():
@@ -55,12 +61,12 @@ def dao_get_organisation_by_service_id(service_id):
    return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()


-@autocommit
+@transactional
 def dao_create_organisation(organisation):
    db.session.add(organisation)


-@autocommit
+@transactional
 def dao_update_organisation(organisation_id, **kwargs):

    domains = kwargs.pop('domains', None)
@@ -83,6 +89,9 @@ def dao_update_organisation(organisation_id, **kwargs):
    if 'organisation_type' in kwargs:
        _update_organisation_services(organisation, 'organisation_type', only_where_none=False)

+    if 'crown' in kwargs:
+        _update_organisation_services(organisation, 'crown', only_where_none=False)
+
    if 'email_branding_id' in kwargs:
        _update_organisation_services(organisation, 'email_branding')

@@ -102,7 +111,7 @@ def _update_organisation_services(organisation, attribute, only_where_none=True)
        db.session.add(service)


-@autocommit
+@transactional
@version_class(Service)
 def dao_add_service_to_organisation(service, organisation_id):
    organisation = Organisation.query.filter_by(
@@ -111,10 +120,15 @@ def dao_add_service_to_organisation(service, organisation_id):

    service.organisation_id = organisation_id
    service.organisation_type = organisation.organisation_type
+    service.crown = organisation.crown

    db.session.add(service)


+def dao_get_invited_organisation_user(user_id):
+    return InvitedOrganisationUser.query.filter_by(id=user_id).one()
+
+
 def dao_get_users_for_organisation(organisation_id):
    return db.session.query(
        User
@@ -126,15 +140,10 @@ def dao_get_users_for_organisation(organisation_id):
    ).order_by(User.created_at).all()


-@autocommit
+@transactional
 def dao_add_user_to_organisation(organisation_id, user_id):
    organisation = dao_get_organisation_by_id(organisation_id)
    user = User.query.filter_by(id=user_id).one()
    user.organisations.append(organisation)
    db.session.add(organisation)
    return user
-
-
-@autocommit
-def dao_remove_user_from_organisation(organisation, user):
-    organisation.users.remove(user)
--- a/app/dao/permissions_dao.py
+++ b/app/dao/permissions_dao.py
@@ -1,16 +1,16 @@
 from app import db
 from app.dao import DAOClass
 from app.models import (
-    MANAGE_API_KEYS,
-    MANAGE_SETTINGS,
-    MANAGE_TEMPLATES,
+    Permission,
    MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
    SEND_EMAILS,
    SEND_LETTERS,
-    SEND_TEXTS,
-    VIEW_ACTIVITY,
-    Permission,
-)
+    MANAGE_API_KEYS,
+    VIEW_ACTIVITY)
+

 # Default permissions for a service
 default_service_permissions = [
--- a/app/dao/provider_details_dao.py
+++ b/app/dao/provider_details_dao.py
@@ -1,18 +1,12 @@
 from datetime import datetime, timedelta

-from flask import current_app
 from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import asc, desc, func
+from flask import current_app

+from app.dao.dao_utils import transactional
+from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    FactBilling,
-    ProviderDetails,
-    ProviderDetailsHistory,
-    User,
-)


 def get_provider_details_by_id(provider_details_id):
@@ -36,8 +30,6 @@ def dao_get_provider_versions(provider_id):
        id=provider_id
    ).order_by(
        desc(ProviderDetailsHistory.version)
-    ).limit(
-        100  # limit results instead of adding pagination
    ).all()


@@ -77,7 +69,7 @@ def _get_sms_providers_for_update(time_threshold):
    return q


-@autocommit
+@transactional
 def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    """
    Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
@@ -86,8 +78,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    amount_to_reduce_by = 10
    providers_list = _get_sms_providers_for_update(time_threshold)

-    if len(providers_list) < 2:
-        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
+    if not providers_list:
        return

    providers = {provider.identifier: provider for provider in providers_list}
@@ -104,7 +95,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
    _adjust_provider_priority(increased_provider, increased_provider_priority)


-@autocommit
+@transactional
 def dao_adjust_provider_priority_back_to_resting_points():
    """
    Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
@@ -138,7 +129,7 @@ def get_provider_details_by_notification_type(notification_type, supports_intern
    return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()


-@autocommit
+@transactional
 def dao_update_provider_details(provider_details):
    _update_provider_details_without_commit(provider_details)

--- a/app/dao/provider_rates_dao.py
+++ b/app/dao/provider_rates_dao.py
@@ -0,0 +1,11 @@
+from app.models import ProviderRates, ProviderDetails
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
+    db.session.add(provider_rates)
--- a/app/dao/returned_letters_dao.py
+++ b/app/dao/returned_letters_dao.py
@@ -1,10 +1,10 @@
 from datetime import datetime

-from sqlalchemy import desc, func
+from sqlalchemy import func, desc
 from sqlalchemy.dialects.postgresql import insert

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import (
    Job,
    Notification,
@@ -28,7 +28,7 @@ def _get_notification_ids_for_references(references):
    return notification_ids + notification_history_ids


-@autocommit
+@transactional
 def insert_or_update_returned_letters(references):
    data = _get_notification_ids_for_references(references)
    for row in data:
--- a/app/dao/service_callback_api_dao.py
+++ b/app/dao/service_callback_api_dao.py
@@ -1,15 +1,13 @@
 from datetime import datetime

-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    COMPLAINT_CALLBACK_TYPE,
-    DELIVERY_STATUS_CALLBACK_TYPE,
-    ServiceCallbackApi,
-)
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
+from app.models import ServiceCallbackApi
+
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE


-@autocommit
+@transactional
@version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
    service_callback_api.id = create_uuid()
@@ -17,7 +15,7 @@ def save_service_callback_api(service_callback_api):
    db.session.add(service_callback_api)


-@autocommit
+@transactional
@version_class(ServiceCallbackApi)
 def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
    if url:
@@ -48,6 +46,6 @@ def get_service_complaint_callback_api_for_service(service_id):
    ).first()


-@autocommit
+@transactional
 def delete_service_callback_api(service_callback_api):
    db.session.delete(service_callback_api)
--- a/app/dao/service_data_retention_dao.py
+++ b/app/dao/service_data_retention_dao.py
@@ -1,7 +1,7 @@
 from datetime import datetime

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceDataRetention


@@ -28,7 +28,7 @@ def fetch_service_data_retention_by_notification_type(service_id, notification_t
    return data_retention_list


-@autocommit
+@transactional
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
    new_data_retention = ServiceDataRetention(service_id=service_id,
                                              notification_type=notification_type,
@@ -38,7 +38,7 @@ def insert_service_data_retention(service_id, notification_type, days_of_retenti
    return new_data_retention


-@autocommit
+@transactional
 def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
    updated_count = ServiceDataRetention.query.filter(
        ServiceDataRetention.id == service_data_retention_id,
@@ -50,7 +50,3 @@ def update_service_data_retention(service_data_retention_id, service_id, days_of
        }
    )
    return updated_count
-
-
-def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
-    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()
--- a/app/dao/service_email_reply_to_dao.py
+++ b/app/dao/service_email_reply_to_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceEmailReplyTo
@@ -28,7 +28,7 @@ def dao_get_reply_to_by_id(service_id, reply_to_id):
    return reply_to


-@autocommit
+@transactional
 def add_reply_to_email_address_for_service(service_id, email_address, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -41,7 +41,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
    return new_reply_to


-@autocommit
+@transactional
 def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -57,7 +57,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
    return reply_to_update


-@autocommit
+@transactional
 def archive_reply_to_email_address(service_id, reply_to_id):
    reply_to_archive = ServiceEmailReplyTo.query.filter_by(
        id=reply_to_id,
--- a/app/dao/service_inbound_api_dao.py
+++ b/app/dao/service_inbound_api_dao.py
@@ -1,11 +1,11 @@
 from datetime import datetime

-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceInboundApi


-@autocommit
+@transactional
@version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
    service_inbound_api.id = create_uuid()
@@ -13,7 +13,7 @@ def save_service_inbound_api(service_inbound_api):
    db.session.add(service_inbound_api)


-@autocommit
+@transactional
@version_class(ServiceInboundApi)
 def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
    if url:
@@ -35,6 +35,6 @@ def get_service_inbound_api_for_service(service_id):
    return ServiceInboundApi.query.filter_by(service_id=service_id).first()


-@autocommit
+@transactional
 def delete_service_inbound_api(service_inbound_api):
    db.session.delete(service_inbound_api)
--- a/app/dao/service_letter_contact_dao.py
+++ b/app/dao/service_letter_contact_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceLetterContact, Template


@@ -30,7 +30,7 @@ def dao_get_letter_contact_by_id(service_id, letter_contact_id):
    return letter_contact


-@autocommit
+@transactional
 def add_letter_contact_for_service(service_id, contact_block, is_default):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -45,7 +45,7 @@ def add_letter_contact_for_service(service_id, contact_block, is_default):
    return new_letter_contact


-@autocommit
+@transactional
 def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
    old_default = _get_existing_default(service_id)
    # if we want to make this the default, ensure there are no other existing defaults
@@ -59,7 +59,7 @@ def update_letter_contact(service_id, letter_contact_id, contact_block, is_defau
    return letter_contact_update


-@autocommit
+@transactional
 def archive_letter_contact(service_id, letter_contact_id):
    letter_contact_to_archive = ServiceLetterContact.query.filter_by(
        id=letter_contact_id,
--- a/app/dao/service_permissions_dao.py
+++ b/app/dao/service_permissions_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServicePermission


@@ -8,7 +8,7 @@ def dao_fetch_service_permissions(service_id):
        ServicePermission.service_id == service_id).all()


-@autocommit
+@transactional
 def dao_add_service_permission(service_id, permission):
    service_permission = ServicePermission(service_id=service_id, permission=permission)
    db.session.add(service_permission)
--- a/app/dao/service_sms_sender_dao.py
+++ b/app/dao/service_sms_sender_dao.py
@@ -1,7 +1,7 @@
 from sqlalchemy import desc

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceSmsSender

@@ -32,7 +32,7 @@ def dao_get_sms_senders_by_service_id(service_id):
    ).order_by(desc(ServiceSmsSender.is_default)).all()


-@autocommit
+@transactional
 def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
    old_default = _get_existing_default(service_id=service_id)
    if is_default:
@@ -51,7 +51,7 @@ def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_n
    return new_sms_sender


-@autocommit
+@transactional
 def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
    old_default = _get_existing_default(service_id)
    if is_default:
@@ -68,7 +68,7 @@ def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default,
    return sms_sender_to_update


-@autocommit
+@transactional
 def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
    service_sms_sender.sms_sender = sms_sender
    service_sms_sender.inbound_number_id = inbound_number_id
@@ -76,7 +76,7 @@ def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sende
    return service_sms_sender


-@autocommit
+@transactional
 def archive_sms_sender(service_id, sms_sender_id):
    sms_sender_to_archive = ServiceSmsSender.query.filter_by(
        id=sms_sender_id,
--- a/app/dao/service_user_dao.py
+++ b/app/dao/service_user_dao.py
@@ -1,6 +1,6 @@

 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceUser, User


@@ -9,13 +9,9 @@ def dao_get_service_user(user_id, service_id):


 def dao_get_active_service_users(service_id):
-    query = db.session.query(
-        ServiceUser
-    ).join(
-        User, User.id == ServiceUser.user_id
-    ).filter(
-        User.state == 'active',
-        ServiceUser.service_id == service_id
+    query = ServiceUser.query.join(ServiceUser.user).filter(
+        ServiceUser.service_id == service_id,
+        User.state == 'active'
    )

    return query.all()
@@ -25,6 +21,6 @@ def dao_get_service_users_by_user_id(user_id):
    return ServiceUser.query.filter_by(user_id=user_id).all()


-@autocommit
+@transactional
 def dao_update_service_user(service_user):
    db.session.add(service_user)
--- a/app/dao/services_dao.py
+++ b/app/dao/services_dao.py
@@ -1,14 +1,18 @@
 import uuid
 from datetime import date, datetime, timedelta

-from flask import current_app
-from sqlalchemy import Float, cast
+from sqlalchemy.sql.expression import asc, case, and_, func
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import and_, asc, case, func
+from sqlalchemy import cast, Float
+from flask import current_app

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
 from app.dao.email_branding_dao import dao_get_email_branding_by_name
 from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
@@ -16,14 +20,6 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    EMAIL_TYPE,
-    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_SMS_TYPE,
-    KEY_TYPE_TEST,
-    LETTER_TYPE,
-    NOTIFICATION_PERMANENT_FAILURE,
-    SMS_TYPE,
-    UPLOAD_LETTERS,
    AnnualBilling,
    ApiKey,
    FactBilling,
@@ -35,22 +31,34 @@ from app.models import (
    Organisation,
    Permission,
    Service,
-    ServiceContactList,
-    ServiceEmailReplyTo,
-    ServiceLetterContact,
    ServicePermission,
    ServiceSmsSender,
+    ServiceEmailReplyTo,
+    ServiceContactList,
+    ServiceLetterContact,
    Template,
    TemplateHistory,
    TemplateRedacted,
    User,
    VerifyCode,
+    CROWN_ORGANISATION_TYPES,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NHS_ORGANISATION_TYPES,
+    NON_CROWN_ORGANISATION_TYPES,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    UPLOAD_LETTERS,
+    INTERNATIONAL_LETTERS
 )
 from app.utils import (
    email_address_is_nhs,
    escape_special_characters,
    get_archived_db_column_value,
    get_london_midnight_in_utc,
+    midnight_n_days_ago,
 )

 DEFAULT_SERVICE_PERMISSIONS = [
@@ -243,7 +251,7 @@ def dao_fetch_all_services_created_by_user(user_id):
    return query.all()


-@autocommit
+@transactional
@version_class(
    VersionOptions(ApiKey, must_write_history=False),
    VersionOptions(Service),
@@ -280,7 +288,7 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
    ).one()


-@autocommit
+@transactional
@version_class(Service)
 def dao_create_service(
    service,
@@ -314,19 +322,27 @@ def dao_create_service(
    if organisation:
        service.organisation_id = organisation.id
        service.organisation_type = organisation.organisation_type
-
        if organisation.email_branding:
            service.email_branding = organisation.email_branding

-        if organisation.letter_branding:
+        if organisation.letter_branding and not service.letter_branding:
            service.letter_branding = organisation.letter_branding

+    elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
+        service.email_branding = dao_get_email_branding_by_name('NHS')
+        service.letter_branding = dao_get_letter_branding_by_name('NHS')
+    if organisation:
+        service.crown = organisation.crown
+    elif service.organisation_type in CROWN_ORGANISATION_TYPES:
+        service.crown = True
+    elif service.organisation_type in NON_CROWN_ORGANISATION_TYPES:
+        service.crown = False
    service.count_as_live = not user.platform_admin

    db.session.add(service)


-@autocommit
+@transactional
@version_class(Service)
 def dao_update_service(service):
    db.session.add(service)
@@ -408,22 +424,46 @@ def delete_service_and_all_associated_db_objects(service):
    db.session.commit()


-def dao_fetch_todays_stats_for_service(service_id):
-    today = date.today()
-    start_date = get_london_midnight_in_utc(today)
+def dao_fetch_stats_for_service(service_id, limit_days):
+    # We always want between seven and eight days
+    start_date = midnight_n_days_ago(limit_days)
+    return _stats_for_service_query(service_id).filter(
+        Notification.created_at >= start_date
+    ).all()

+
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def fetch_todays_total_message_count(service_id):
+    result = db.session.query(
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        func.date(Notification.created_at) == date.today()
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).first()
+    return 0 if result is None else result.count
+
+
+def _stats_for_service_query(service_id):
    return db.session.query(
        Notification.notification_type,
        Notification.status,
        func.count(Notification.id).label('count')
    ).filter(
        Notification.service_id == service_id,
-        Notification.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= start_date
+        Notification.key_type != KEY_TYPE_TEST
    ).group_by(
        Notification.notification_type,
        Notification.status,
-    ).all()
+    )


 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
@@ -471,7 +511,7 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act
    return query.all()


-@autocommit
+@transactional
@version_class(
    VersionOptions(ApiKey, must_write_history=False),
    VersionOptions(Service),
@@ -490,7 +530,7 @@ def dao_suspend_service(service_id):
    service.active = False


-@autocommit
+@transactional
@version_class(Service)
 def dao_resume_service(service_id):
    service = Service.query.get(service_id)
@@ -518,8 +558,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
        Notification.notification_type == SMS_TYPE,
        func.substr(Notification.normalised_to, 3, 7) == '7700900',
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
    ).having(
@@ -527,7 +567,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
    ).all()


-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
    subquery = db.session.query(
        func.count(Notification.id).label('total_count'),
        Notification.service_id.label('service_id')
@@ -538,8 +578,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
        Notification.key_type != KEY_TYPE_TEST,
        Notification.notification_type == SMS_TYPE,
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
    ).having(
@@ -564,8 +604,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
        Notification.notification_type == SMS_TYPE,
        Notification.status == NOTIFICATION_PERMANENT_FAILURE,
        Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
    ).group_by(
        Notification.service_id,
        subquery.c.total_count
@@ -574,23 +614,3 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
    )

    return query.all()
-
-
-def get_live_services_with_organisation():
-    query = db.session.query(
-        Service.id.label("service_id"),
-        Service.name.label("service_name"),
-        Organisation.id.label("organisation_id"),
-        Organisation.name.label("organisation_name")
-    ).outerjoin(
-        Service.organisation
-    ).filter(
-        Service.count_as_live.is_(True),
-        Service.active.is_(True),
-        Service.restricted.is_(False)
-    ).order_by(
-        Organisation.name,
-        Service.name
-    )
-
-    return query.all()
--- a/app/dao/template_folder_dao.py
+++ b/app/dao/template_folder_dao.py
@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import TemplateFolder


@@ -14,16 +14,16 @@ def dao_get_valid_template_folders_by_id(folder_ids):
    return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all()


-@autocommit
+@transactional
 def dao_create_template_folder(template_folder):
    db.session.add(template_folder)


-@autocommit
+@transactional
 def dao_update_template_folder(template_folder):
    db.session.add(template_folder)


-@autocommit
+@transactional
 def dao_delete_template_folder(template_folder):
    db.session.delete(template_folder)
--- a/app/dao/templates_dao.py
+++ b/app/dao/templates_dao.py
@@ -1,22 +1,26 @@
-import uuid
 from datetime import datetime
+import uuid

 from flask import current_app
 from sqlalchemy import asc, desc

 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.users_dao import get_user_by_id
 from app.models import (
    LETTER_TYPE,
    SECOND_CLASS,
    Template,
    TemplateHistory,
-    TemplateRedacted,
+    TemplateRedacted
 )
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.users_dao import get_user_by_id


-@autocommit
+@transactional
@version_class(
    VersionOptions(Template, history_class=TemplateHistory)
 )
@@ -38,15 +42,18 @@ def dao_create_template(template):
    db.session.add(template)


-@autocommit
+@transactional
@version_class(
    VersionOptions(Template, history_class=TemplateHistory)
 )
 def dao_update_template(template):
+    if template.archived:
+        template.folder = None
+
    db.session.add(template)


-@autocommit
+@transactional
 def dao_update_template_reply_to(template_id, reply_to):
    Template.query.filter_by(id=template_id).update(
        {"service_letter_contact_id": reply_to,
@@ -78,7 +85,7 @@ def dao_update_template_reply_to(template_id, reply_to):
    return template


-@autocommit
+@transactional
 def dao_redact_template(template, user_id):
    template.template_redacted.redact_personalisation = True
    template.template_redacted.updated_at = datetime.utcnow()
--- a/app/dao/uploads_dao.py
+++ b/app/dao/uploads_dao.py
@@ -1,18 +1,11 @@
 from datetime import datetime
-
 from flask import current_app
-from sqlalchemy import String, and_, desc, func, literal, text
+from sqlalchemy import and_, desc, func, literal, text, String

 from app import db
 from app.models import (
-    JOB_STATUS_CANCELLED,
-    JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    Job,
-    Notification,
-    ServiceDataRetention,
-    Template,
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED, ServiceDataRetention
 )
 from app.utils import midnight_n_days_ago

@@ -50,8 +43,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
        func.coalesce(
            Job.processing_started, Job.created_at
-        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
-        Job.contact_list_id.is_(None),
+        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
    ]
    if limit_days is not None:
        jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
@@ -84,7 +76,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
        Notification.notification_type == LETTER_TYPE,
        Notification.api_key_id == None,  # noqa
        Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
+        Template.hidden == True,
        Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
    ]
    if limit_days is not None:
--- a/app/dao/users_dao.py
+++ b/app/dao/users_dao.py
@@ -1,16 +1,16 @@
+from random import (SystemRandom)
+from datetime import (datetime, timedelta)
 import uuid
-from datetime import datetime, timedelta
-from random import SystemRandom

 from sqlalchemy import func
 from sqlalchemy.orm import joinedload

 from app import db
-from app.dao.dao_utils import autocommit
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_users_by_user_id
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
-from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
+from app.models import (EMAIL_AUTH_TYPE, User, VerifyCode)
 from app.utils import escape_special_characters, get_archived_db_column_value


@@ -20,23 +20,15 @@ def _remove_values_for_keys_if_present(dict, keys):


 def create_secret_code():
-    return ''.join(get_non_repeating_random_digits(5))
+    return ''.join(map(str, [SystemRandom().randrange(10) for i in range(5)]))


-def get_non_repeating_random_digits(length):
-    output = [None] * length
-    for index in range(length):
-        while output[index] in {None, output[index - 1]}:
-            output[index] = str(SystemRandom().randrange(10))
-    return output
-
-
-def save_user_attribute(usr, update_dict=None):
-    db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
+def save_user_attribute(usr, update_dict={}):
+    db.session.query(User).filter_by(id=usr.id).update(update_dict)
    db.session.commit()


-def save_model_user(user, update_dict=None, password=None, validated_email_access=False):
+def save_model_user(user, update_dict={}, password=None, validated_email_access=False):
    if password:
        user.password = password
        user.password_changed_at = datetime.utcnow()
@@ -44,7 +36,7 @@ def save_model_user(user, update_dict=None, password=None, validated_email_acces
        user.email_access_validated_at = datetime.utcnow()
    if update_dict:
        _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
-        db.session.query(User).filter_by(id=user.id).update(update_dict or {})
+        db.session.query(User).filter_by(id=user.id).update(update_dict)
    else:
        db.session.add(user)
    db.session.commit()
@@ -131,10 +123,12 @@ def reset_failed_login_count(user):
        db.session.commit()


-def update_user_password(user, password):
+def update_user_password(user, password, validated_email_access=False):
    # reset failed login count - they've just reset their password so should be fine
    user.password = password
    user.password_changed_at = datetime.utcnow()
+    if validated_email_access:
+        user.email_access_validated_at = datetime.utcnow()
    db.session.add(user)
    db.session.commit()

@@ -152,7 +146,7 @@ def get_user_and_accounts(user_id):
    ).one()


-@autocommit
+@transactional
 def dao_archive_user(user):
    if not user_can_be_archived(user):
        msg = "User can’t be removed from a service - check all services have another team member with manage_settings"
--- a/app/dao/webauthn_credential_dao.py
+++ b/app/dao/webauthn_credential_dao.py
@@ -1,40 +0,0 @@
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import WebauthnCredential
-
-
-def dao_get_webauthn_credential_by_user_and_id(user_id, webauthn_credential_id):
-    return WebauthnCredential.query.filter(
-        WebauthnCredential.user_id == user_id,
-        WebauthnCredential.id == webauthn_credential_id
-    ).one()
-
-
-@autocommit
-def dao_create_webauthn_credential(
-    *,
-    user_id,
-    name,
-    credential_data,
-    registration_response,
-):
-    webauthn_credential = WebauthnCredential(
-        user_id=user_id,
-        name=name,
-        credential_data=credential_data,
-        registration_response=registration_response
-    )
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_update_webauthn_credential_name(webauthn_credential, new_name):
-    webauthn_credential.name = new_name
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_delete_webauthn_credential(webauthn_credential):
-    db.session.delete(webauthn_credential)
--- a/app/delivery/send_to_providers.py
+++ b/app/delivery/send_to_providers.py
@@ -1,43 +1,40 @@
 import random
-from datetime import datetime, timedelta
 from urllib import parse
+from datetime import datetime, timedelta

-from cachetools import TTLCache, cached
 from flask import current_app
-from notifications_utils.template import (
-    HTMLEmailTemplate,
-    PlainTextEmailTemplate,
-    SMSMessageTemplate,
+from notifications_utils.recipients import (
+    validate_and_format_phone_number,
+    validate_and_format_email_address
 )
+from notifications_utils.template import HTMLEmailTemplate, PlainTextEmailTemplate, SMSMessageTemplate

-from app import create_uuid, db, notification_provider_clients, statsd_client
-from app.celery.research_mode_tasks import (
-    send_email_response,
-    send_sms_response,
+from app import clients, statsd_client, create_uuid
+from app.dao.notifications_dao import (
+    dao_update_notification
 )
-from app.dao.email_branding_dao import dao_get_email_branding_by_id
-from app.dao.notifications_dao import dao_update_notification
 from app.dao.provider_details_dao import (
-    dao_reduce_sms_provider_priority,
    get_provider_details_by_notification_type,
+    dao_reduce_sms_provider_priority
 )
+from app.celery.research_mode_tasks import send_sms_response, send_email_response
+from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
+    SMS_TYPE,
+    KEY_TYPE_TEST,
    BRANDING_BOTH,
    BRANDING_ORG_BANNER,
    EMAIL_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
    NOTIFICATION_TECHNICAL_FAILURE,
-    SMS_TYPE,
+    NOTIFICATION_SENT,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_STATUS_TYPES_COMPLETED
 )
-from app.serialised_models import SerialisedService, SerialisedTemplate


 def send_sms_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
+    service = notification.service

    if not service.active:
        technical_failure(notification=notification)
@@ -45,13 +42,8 @@ def send_sms_to_provider(notification):

    if notification.status == 'created':
        provider = provider_to_use(SMS_TYPE, notification.international)
-        if not provider:
-            technical_failure(notification=notification)
-            return

-        template_model = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        )
+        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)

        template = SMSMessageTemplate(
            template_model.__dict__,
@@ -59,63 +51,41 @@ def send_sms_to_provider(notification):
            prefix=service.name,
            show_prefix=service.prefix_sms,
        )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
            update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.get_name(), str(notification.id), notification.to)

        else:
            try:
-                # End DB session here so that we don't have a connection stuck open waiting on the call
-                # to one of the SMS providers
-                # We don't want to tie our DB connections being open to the performance of our SMS
-                # providers as a slow down of our providers can cause us to run out of DB connections
-                # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
-                # closing the session (as otherwise it would be reopened immediately)
-                send_sms_kwargs = {
-                    'to': notification.normalised_to,
-                    'content': str(template),
-                    'reference': str(notification.id),
-                    'sender': notification.reply_to_text,
-                    'international': notification.international,
-                }
-                db.session.close()  # no commit needed as no changes to objects have been made above
-                provider.send_sms(**send_sms_kwargs)
+                provider.send_sms(
+                    to=validate_and_format_phone_number(notification.to, international=notification.international),
+                    content=str(template),
+                    reference=str(notification.id),
+                    sender=notification.reply_to_text
+                )
            except Exception as e:
                notification.billable_units = template.fragment_count
                dao_update_notification(notification)
-                dao_reduce_sms_provider_priority(provider.name, time_threshold=timedelta(minutes=1))
+                dao_reduce_sms_provider_priority(provider.get_name(), time_threshold=timedelta(minutes=1))
                raise e
            else:
                notification.billable_units = template.fragment_count
                update_notification_to_sending(notification, provider)

-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
        statsd_client.timing("sms.total-time", delta_seconds)

-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("sms.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("sms.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("sms.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("sms.live-key.not-high-volume.total-time", delta_seconds)
-

 def send_email_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
-
+    service = notification.service
    if not service.active:
        technical_failure(notification=notification)
        return
    if notification.status == 'created':
        provider = provider_to_use(EMAIL_TYPE)

-        template_dict = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        ).__dict__
+        template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__

        html_email = HTMLEmailTemplate(
            template_dict,
@@ -127,8 +97,7 @@ def send_email_to_provider(notification):
            template_dict,
            values=notification.personalisation
        )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
        if service.research_mode or notification.key_type == KEY_TYPE_TEST:
            notification.reference = str(create_uuid())
            update_notification_to_sending(notification, provider)
@@ -137,48 +106,34 @@ def send_email_to_provider(notification):
            from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
                                                 current_app.config['NOTIFY_EMAIL_DOMAIN'])

+            email_reply_to = notification.reply_to_text
+
            reference = provider.send_email(
                from_address,
-                notification.normalised_to,
+                validate_and_format_email_address(notification.to),
                plain_text_email.subject,
                body=str(plain_text_email),
                html_body=str(html_email),
-                reply_to_address=notification.reply_to_text
+                reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
            )
            notification.reference = reference
            update_notification_to_sending(notification, provider)
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()

-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("email.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("email.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
+        statsd_client.timing("email.total-time", delta_seconds)


 def update_notification_to_sending(notification, provider):
    notification.sent_at = datetime.utcnow()
-    notification.sent_by = provider.name
+    notification.sent_by = provider.get_name()
    if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
-        # We currently have no callback method for SMS deliveries
-        # TODO create celery task to request SMS delivery receipts from cloudwatch api
-        notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING
-    
+        notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
    dao_update_notification(notification)


-provider_cache = TTLCache(maxsize=8, ttl=10)
-
-
-@cached(cache=provider_cache)
-def provider_to_use(notification_type, international=True):
-    international = False # TODO: remove or resolve the functionality of this flag
-    # TODO rip firetext and mmg out of early migrations and clean up the expression below
+def provider_to_use(notification_type, international=False):
    active_providers = [
-        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg']
+        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active
    ]

    if not active_providers:
@@ -187,13 +142,9 @@ def provider_to_use(notification_type, international=True):
        )
        raise Exception("No active {} providers".format(notification_type))

-    if len(active_providers) == 1:
-        chosen_provider = active_providers[0]
-    else:
-        weights = [p.priority for p in active_providers]
-        chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related
+    chosen_provider = random.choices(active_providers, weights=[p.priority for p in active_providers])[0]

-    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
+    return clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)


 def get_logo_url(base_url, logo_file):
@@ -218,28 +169,25 @@ def get_logo_url(base_url, logo_file):


 def get_html_email_options(service):
+
    if service.email_branding is None:
        return {
            'govuk_banner': True,
            'brand_banner': False,
        }
-    if isinstance(service, SerialisedService):
-        branding = dao_get_email_branding_by_id(service.email_branding)
-    else:
-        branding = service.email_branding

    logo_url = get_logo_url(
        current_app.config['ADMIN_BASE_URL'],
-        branding.logo
-    ) if branding.logo else None
+        service.email_branding.logo
+    ) if service.email_branding.logo else None

    return {
-        'govuk_banner': branding.brand_type == BRANDING_BOTH,
-        'brand_banner': branding.brand_type == BRANDING_ORG_BANNER,
-        'brand_colour': branding.colour,
+        'govuk_banner': service.email_branding.brand_type == BRANDING_BOTH,
+        'brand_banner': service.email_branding.brand_type == BRANDING_ORG_BANNER,
+        'brand_colour': service.email_branding.colour,
        'brand_logo': logo_url,
-        'brand_text': branding.text,
-        'brand_name': branding.name,
+        'brand_text': service.email_branding.text,
+        'brand_name': service.email_branding.name,
    }


--- a/app/email_branding/email_branding_schema.py
+++ b/app/email_branding/email_branding_schema.py
@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES

 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST schema for getting email_branding",
    "type": "object",
    "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }

 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
    "description": "POST schema for getting email_branding",
    "type": "object",
    "properties": {
--- a/app/email_branding/rest.py
+++ b/app/email_branding/rest.py
@@ -2,16 +2,16 @@ from flask import Blueprint, jsonify, request

 from app.dao.email_branding_dao import (
    dao_create_email_branding,
-    dao_get_email_branding_by_id,
    dao_get_email_branding_options,
-    dao_update_email_branding,
-)
-from app.email_branding.email_branding_schema import (
-    post_create_email_branding_schema,
-    post_update_email_branding_schema,
+    dao_get_email_branding_by_id,
+    dao_update_email_branding
 )
 from app.errors import register_errors
 from app.models import EmailBranding
+from app.email_branding.email_branding_schema import (
+    post_create_email_branding_schema,
+    post_update_email_branding_schema
+)
 from app.schema_validation import validate

 email_branding_blueprint = Blueprint('email_branding', __name__)
--- a/app/errors.py
+++ b/app/errors.py
@@ -1,10 +1,12 @@
-from flask import current_app, json, jsonify
-from jsonschema import ValidationError as JsonSchemaValidationError
-from marshmallow import ValidationError
+from flask import (
+    jsonify,
+    current_app,
+    json)
 from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
-
+from marshmallow import ValidationError
+from jsonschema import ValidationError as JsonSchemaValidationError
 from app.authentication.auth import AuthError
 from app.exceptions import ArchiveValidationError

--- a/app/events/rest.py
+++ b/app/events/rest.py
@@ -1,8 +1,13 @@
-from flask import Blueprint, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)

-from app.dao.events_dao import dao_create_event
 from app.errors import register_errors
+
 from app.schemas import event_schema
+from app.dao.events_dao import dao_create_event

 events = Blueprint('events', __name__, url_prefix='/events')
 register_errors(events)
@@ -11,6 +16,6 @@ register_errors(events)
@events.route('', methods=['POST'])
 def create_event():
    data = request.get_json()
-    event = event_schema.load(data)
+    event = event_schema.load(data).data
    dao_create_event(event)
-    return jsonify(data=event_schema.dump(event)), 201
+    return jsonify(data=event_schema.dump(event).data), 201
--- a/app/govuk_alerts/rest.py
+++ b/app/govuk_alerts/rest.py
@@ -1,31 +0,0 @@
-from flask import Blueprint, jsonify
-
-from app.dao.broadcast_message_dao import dao_get_all_broadcast_messages
-from app.errors import register_errors
-from app.utils import get_dt_string_or_none
-
-govuk_alerts_blueprint = Blueprint(
-    "govuk-alerts",
-    __name__,
-    url_prefix='/govuk-alerts',
-)
-
-register_errors(govuk_alerts_blueprint)
-
-
-@govuk_alerts_blueprint.route('')
-def get_broadcasts():
-    broadcasts = dao_get_all_broadcast_messages()
-    broadcasts_dict = {"alerts": [{
-        "id": broadcast.id,
-        "reference": broadcast.reference,
-        "channel": broadcast.channel,
-        "content": broadcast.content,
-        "areas": broadcast.areas,
-        "status": broadcast.status,
-        "starts_at": get_dt_string_or_none(broadcast.starts_at),
-        "finishes_at": get_dt_string_or_none(broadcast.finishes_at),
-        "approved_at": get_dt_string_or_none(broadcast.approved_at),
-        "cancelled_at": get_dt_string_or_none(broadcast.cancelled_at),
-    } for broadcast in broadcasts]}
-    return jsonify(broadcasts_dict), 200
--- a/app/hashing.py
+++ b/app/hashing.py
@@ -1,4 +1,4 @@
-from flask_bcrypt import check_password_hash, generate_password_hash
+from flask_bcrypt import generate_password_hash, check_password_hash


 def hashpw(password):
--- a/app/history_meta.py
+++ b/app/history_meta.py
@@ -15,11 +15,11 @@ session events.

 """
 import datetime
-
-from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, util
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import attributes, mapper, object_mapper
-from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty
+from sqlalchemy.orm import mapper, attributes, object_mapper
+from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty
+from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
+from sqlalchemy import util


 def col_references_table(col, table):
--- a/Show More
+++ b/Show More