Add workers for sms and email sending

We want to have different workers for sending out email and sending out
sms. This will enable us to better rate limit ourselves when sending to
our providers. For example, we can make sure we only have capacity in
the email worker to send at our rate of 500rps to SES.

Note, we may in due time need to split the sms sender worker into two
different workers, one for each provider.

A separate PR will come after this to remove the unsplit
`notify-delivery-worker-sender`.

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
+# W503: line break before binary operator
+exclude = venv*,__pycache__,node_modules,cache,migrations,build
+ignore = W503
+max-complexity = 14
+max-line-length = 120

.gitignore

@@ -69,8 +69,6 @@ target/
 *.DS_Store
 environment.sh
 .envrc
-.env
-.env*
 
 celerybeat-schedule
 

.pyup.yml

@@ -4,5 +4,5 @@ schedule: "every week on wednesday"
 
 search: False
 requirements:
-  - requirements.in
-  - requirements_for_test.txt
+  - requirements-app.txt
+  - requirements-dev.txt

Makefile

@@ -7,57 +7,28 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
+
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
+
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
+
 CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
-CF_MANIFEST_PATH ?= /tmp/manifest.yml
-
+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
 
 NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
 
 ## DEVELOPMENT
 
-.PHONY: bootstrap
-bootstrap: generate-version-file ## Set up everything to run the app
-	pip3 install -r requirements_for_test.txt
-	createdb notification_api || true
-	(flask db upgrade) || true
-
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
-
-.PHONY: run-flask
-run-flask: ## Run flask
-	flask run -p 6011 --host=0.0.0.0
-
-.PHONY: run-celery
-run-celery: ## Run celery, TODO remove purge for staging/prod
-	celery -A run_celery.notify_celery purge -f
-	celery \
-		-A run_celery.notify_celery worker \
-		--pidfile="/tmp/celery.pid" \
-		--loglevel=INFO \
-		--concurrency=4
-
-.PHONY: run-celery-with-docker
-run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery
-
-.PHONY: run-celery-beat
-run-celery-beat: ## Run celery beat
-	celery \
-	-A run_celery.notify_celery beat \
-	--loglevel=INFO
-
-.PHONY: run-celery-beat-with-docker
-run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally)
-	./scripts/run_with_docker.sh make run-celery-beat
-
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -67,19 +38,71 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: test
-test: ## Run tests
-	flake8 .
-	isort --check-only ./app ./tests
-	pytest -n4 --maxfail=10
+test: generate-version-file ## Run tests
+	./scripts/run_tests.sh
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pip install --upgrade pip-tools
-	pip-compile requirements.in
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
+
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
+
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
+	docker build -f docker/Dockerfile \
+		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
+		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
+		--build-arg NO_PROXY="${NO_PROXY}" \
+		-t ${DOCKER_BUILDER_IMAGE_NAME} \
+		.
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	@docker run -it --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
+		-e GIT_COMMIT=${GIT_COMMIT} \
+		-e BUILD_NUMBER=${BUILD_NUMBER} \
+		-e BUILD_URL=${BUILD_URL} \
+		-e http_proxy="${HTTP_PROXY}" \
+		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e https_proxy="${HTTPS_PROXY}" \
+		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e NO_PROXY="${NO_PROXY}" \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: create-docker-test-db
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	@docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
 
 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
+	rm -rf node_modules cache target venv .coverage build tests/.cache
 
 
 ## DEPLOYMENT
@@ -131,27 +154,18 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 	@cf app --guid ${CF_APP} || exit 1
 
 	# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
-	cf cancel-deployment ${CF_APP} || true
+	cf v3-cancel-zdt-push ${CF_APP} || true
 
-	# generate manifest (including secrets) and write it to CF_MANIFEST_PATH (in /tmp/)
-	make -s CF_APP=${CF_APP} generate-manifest > ${CF_MANIFEST_PATH}
+	cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
+	CF_STARTUP_TIMEOUT=15 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete  # fails after 15 mins if deploy doesn't work
 
-	# fails after 15 mins if deploy doesn't work
-	# reads manifest from CF_MANIFEST_PATH
-	CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling -f ${CF_MANIFEST_PATH}
-	# delete old manifest file
-	rm ${CF_MANIFEST_PATH}
 
 .PHONY: cf-deploy-api-db-migration
 cf-deploy-api-db-migration:
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
-	make -s CF_APP=notify-api-db-migration generate-manifest > ${CF_MANIFEST_PATH}
-
-	cf push notify-api-db-migration --no-route -f ${CF_MANIFEST_PATH}
-	rm ${CF_MANIFEST_PATH}
-
-	cf run-task notify-api-db-migration --command="flask db upgrade" --name api_db_migration
+	cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
+	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
 
 .PHONY: cf-check-api-db-migration-task
 cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
@@ -160,15 +174,14 @@ cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-mig
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
 	$(if ${CF_APP},,$(error Must specify CF_APP))
-	rm ${CF_MANIFEST_PATH}
-	cf cancel-deployment ${CF_APP}
+	cf v3-cancel-zdt-push ${CF_APP}
 
 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:
 	@echo $(shell python3 scripts/check_if_new_migration.py)
 
 .PHONY: cf-deploy-failwhale
-cf-deploy-failwhale:
+cf-deploy-failwhale:  #
 	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
 	cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
 

README.md

@@ -9,19 +9,25 @@ Contains:
 
 ### Python version
 
-We run python 3.9 both locally and in production.
-
-### psycopg2
-
-[Follow these instructions on Mac M1 machines](https://github.com/psycopg/psycopg2/issues/1216#issuecomment-1068150544).
+At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.
 
 ### AWS credentials
 
-To run the API you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details.
+To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.
+
+Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
+
+### Virtualenv
+
+```
+mkvirtualenv -p /usr/local/bin/python3 notifications-api
+```
 
 ### `environment.sh`
 
-Creating and edit an environment.sh file.
+Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
+
+Create a local environment.sh file containing the following:
 
 ```
 echo "
@@ -32,87 +38,81 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
 export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
 
 export FLASK_APP=application.py
-export FLASK_ENV=development
+export FLASK_DEBUG=1
 export WERKZEUG_DEBUG_PIN=off
 "> environment.sh
 ```
 
-Things to change:
+NOTES:
 
-* Replace `YOUR_OWN_PREFIX` with `local_dev_<first name>`.
-* Run the following in the credentials repo to get the API keys.
-
-```
-notify-pass credentials/firetext
-notify-pass credentials/mmg
-```
+ * Replace the placeholder key and prefix values as appropriate
+ * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
+ * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.
 
 ### Postgres
 
-Install [Postgres.app](http://postgresapp.com/).
+Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
 
-Currently the API works with PostgreSQL 11. After installation, open the Postgres app, open the sidebar, and update or replace the default server with a compatible version.
-
-**Note:** you may need to add the following directory to your PATH in order to bootstrap the app.
-
-```
-export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/
-```
+Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.
 
 ### Redis
 
-To switch redis on you'll need to install it locally. On a Mac you can do:
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
 
-```
-# assuming you use Homebrew
-brew install redis
-brew services start redis
-```
+        REDIS_ENABLED = True
 
-To use redis caching you need to switch it on with an environment variable:
-
-```
-export REDIS_ENABLED=1
-```
 
 ##  To run the application
 
-```
-# install dependencies, etc.
-make bootstrap
+First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
 
-# run the web app
-make run-flask
+You need to run the api application and a local celery instance.
 
-# run the background tasks
-make run-celery
-
-# run scheduled tasks (optional)
-make run-celery-beat
-```
-
-We've had problems running Celery locally due to one of its dependencies: pycurl. Due to the complexity of the issue, we also support running Celery via Docker:
+There are two run scripts for running all the necessary parts.
 
 ```
-# install dependencies, etc.
-make bootstrap-with-docker
-
-# run the background tasks
-make run-celery-with-docker
-
-# run scheduled tasks
-make run-celery-beat-with-docker
+scripts/run_app.sh
 ```
 
+```
+scripts/run_celery.sh
+```
+
+Optionally you can also run this script to run the scheduled tasks:
+
+```
+scripts/run_celery_beat.sh
+```
+
+
 ##  To test the application
 
-```
-# install dependencies, etc.
-make bootstrap
+First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
 
+Then simply run
+
+```
 make test
 ```
 
+That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
+[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
+
+
+## To update application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with
+
+```
+make freeze-requirements
+```
+
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.
+
+
 ## To run one off tasks
 
 Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
@@ -131,7 +131,28 @@ cf run-task notify-api "flask command purge_functional_test_data -u <functional
 
 All commands and command options have a --help command if you need more information.
 
-## Further documentation
 
-- [Writing public APIs](docs/writing-public-apis.md)
-- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
+## To create a new worker app
+
+You need to:
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
+1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
+1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
+1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
+
+**Important:**
+
+Before pushing the deployment change on jenkins, read below about the first time deployment.
+
+### First time deployment of your new worker
+
+Our deployment flow requires that the app is present in order to proceed with the deployment.
+
+This means that the first deployment of your app must happen manually.
+
+To do this:
+
+1. Ensure your code is backwards compatible
+1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
+
+Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.

app/__init__.py

@@ -1,40 +1,38 @@
+import time
 import os
 import random
 import string
-import time
 import uuid
-from time import monotonic
 
 from celery import current_task
-from flask import (
-    current_app,
-    g,
-    has_request_context,
-    jsonify,
-    make_response,
-    request,
-)
+from flask import _request_ctx_stack, request, g, jsonify, make_response, current_app, has_request_context
+from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from gds_metrics import GDSMetrics
 from gds_metrics.metrics import Gauge, Histogram
-from notifications_utils import logging, request_helper
-from notifications_utils.celery import NotifyCelery
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from time import monotonic
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
+from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils import logging, request_helper
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy
 
-from app.clients import NotificationProviderClients
-from app.clients.cbc_proxy import CBCProxyClient
+from app.celery.celery import NotifyCelery
+from app.clients import Clients
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.aws_sns import AwsSnsClient
+from app.clients.sms.firetext import FiretextClient
+from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+
+DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+DATE_FORMAT = "%Y-%m-%d"
 
 
 class SQLAlchemy(_SQLAlchemy):
@@ -53,21 +51,22 @@ db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
+firetext_client = FiretextClient()
+mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 aws_ses_stub_client = AwsSesStubClient()
-aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
-cbc_proxy_client = CBCProxyClient()
+performance_platform_client = PerformancePlatformClient()
 document_download_client = DocumentDownloadClient()
 metrics = GDSMetrics()
 
-notification_provider_clients = NotificationProviderClients()
+clients = Clients()
 
-api_user = LocalProxy(lambda: g.api_user)
-authenticated_service = LocalProxy(lambda: g.authenticated_service)
+api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
+authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
 
 CONCURRENT_REQUESTS = Gauge(
     'concurrent_web_request_count',
@@ -94,7 +93,8 @@ def create_app(application):
     zendesk_client.init_app(application)
     statsd_client.init_app(application)
     logging.init_app(application, statsd_client)
-    aws_sns_client.init_app(application, statsd_client=statsd_client)
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application, statsd_client=statsd_client)
 
     aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     aws_ses_stub_client.init_app(
@@ -104,18 +104,14 @@ def create_app(application):
     )
     # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
     email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
-    notification_provider_clients.init_app(
-        sms_clients=[aws_sns_client],
-        email_clients=email_clients
-    )
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=email_clients)
 
     notify_celery.init_app(application)
     encryption.init_app(application)
     redis_store.init_app(application)
+    performance_platform_client.init_app(application)
     document_download_client.init_app(application)
 
-    cbc_proxy_client.init_app(application)
-
     register_blueprint(application)
     register_v2_blueprints(application)
 
@@ -130,56 +126,35 @@ def create_app(application):
 
 
 def register_blueprint(application):
-    from app.authentication.auth import (
-        requires_admin_auth,
-        requires_auth,
-        requires_govuk_alerts_auth,
-        requires_no_auth,
-    )
-    from app.billing.rest import billing_blueprint
-    from app.broadcast_message.rest import broadcast_message_blueprint
-    from app.complaint.complaint_rest import complaint_blueprint
-    from app.email_branding.rest import email_branding_blueprint
+    from app.service.rest import service_blueprint
+    from app.service.callback_rest import service_callback_blueprint
+    from app.user.rest import user_blueprint
+    from app.template.rest import template_blueprint
+    from app.status.healthcheck import status as status_blueprint
+    from app.job.rest import job_blueprint
+    from app.notifications.rest import notifications as notifications_blueprint
+    from app.invite.rest import invite as invite_blueprint
+    from app.accept_invite.rest import accept_invite
+    from app.template_statistics.rest import template_statistics as template_statistics_blueprint
     from app.events.rest import events as events_blueprint
-    from app.govuk_alerts.rest import govuk_alerts_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.email_branding.rest import email_branding_blueprint
     from app.inbound_number.rest import inbound_number_blueprint
     from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
-    from app.job.rest import job_blueprint
-    from app.letter_branding.letter_branding_rest import (
-        letter_branding_blueprint,
-    )
+    from app.notifications.receive_notifications import receive_notifications_blueprint
+    from app.notifications.notifications_sms_callback import sms_callback_blueprint
+    from app.notifications.notifications_letter_callback import letter_callback_blueprint
+    from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
     from app.letters.rest import letter_job
-    from app.notifications.notifications_letter_callback import (
-        letter_callback_blueprint,
-    )
-    from app.notifications.notifications_sms_callback import (
-        sms_callback_blueprint,
-    )
-    from app.notifications.receive_notifications import (
-        receive_notifications_blueprint,
-    )
-    from app.notifications.rest import notifications as notifications_blueprint
-    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.billing.rest import billing_blueprint
     from app.organisation.rest import organisation_blueprint
-    from app.performance_dashboard.rest import performance_dashboard_blueprint
+    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.complaint.complaint_rest import complaint_blueprint
     from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import (
-        provider_details as provider_details_blueprint,
-    )
-    from app.service.callback_rest import service_callback_blueprint
-    from app.service.rest import service_blueprint
-    from app.service_invite.rest import (
-        service_invite as service_invite_blueprint,
-    )
-    from app.status.healthcheck import status as status_blueprint
-    from app.template.rest import template_blueprint
     from app.template_folder.rest import template_folder_blueprint
-    from app.template_statistics.rest import (
-        template_statistics as template_statistics_blueprint,
-    )
+    from app.letter_branding.letter_branding_rest import letter_branding_blueprint
     from app.upload.rest import upload_blueprint
-    from app.user.rest import user_blueprint
-    from app.webauthn.rest import webauthn_blueprint
+    from app.broadcast_message.rest import broadcast_message_blueprint
 
     service_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -187,9 +162,6 @@ def register_blueprint(application):
     user_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(user_blueprint, url_prefix='/user')
 
-    webauthn_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(webauthn_blueprint)
-
     template_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_blueprint)
 
@@ -211,11 +183,8 @@ def register_blueprint(application):
     job_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(job_blueprint)
 
-    service_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_invite_blueprint)
-
-    organisation_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organisation_invite_blueprint)
+    invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(invite_blueprint)
 
     inbound_number_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_number_blueprint)
@@ -223,6 +192,9 @@ def register_blueprint(application):
     inbound_sms_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_sms_blueprint)
 
+    accept_invite.before_request(requires_admin_auth)
+    application.register_blueprint(accept_invite, url_prefix='/invite')
+
     template_statistics_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_statistics_blueprint)
 
@@ -250,12 +222,12 @@ def register_blueprint(application):
     organisation_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(organisation_blueprint, url_prefix='/organisations')
 
+    organisation_invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(organisation_invite_blueprint)
+
     complaint_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(complaint_blueprint)
 
-    performance_dashboard_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(performance_dashboard_blueprint)
-
     platform_stats_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')
 
@@ -271,40 +243,33 @@ def register_blueprint(application):
     broadcast_message_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(broadcast_message_blueprint)
 
-    govuk_alerts_blueprint.before_request(requires_govuk_alerts_auth)
-    application.register_blueprint(govuk_alerts_blueprint)
-
 
 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
+    from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
+    from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
+    from app.v2.template.get_template import v2_template_blueprint as get_template
+    from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
+    from app.v2.template.post_template import v2_template_blueprint as post_template
     from app.authentication.auth import requires_auth
-    from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint
 
-    v2_notification_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
+    post_notifications.before_request(requires_auth)
+    application.register_blueprint(post_notifications)
 
-    v2_templates_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
+    get_notifications.before_request(requires_auth)
+    application.register_blueprint(get_notifications)
 
-    v2_template_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
+    get_templates.before_request(requires_auth)
+    application.register_blueprint(get_templates)
 
-    v2_inbound_sms_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
+    get_template.before_request(requires_auth)
+    application.register_blueprint(get_template)
 
-    v2_broadcast_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_broadcast_blueprint)
+    post_template.before_request(requires_auth)
+    application.register_blueprint(post_template)
+
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)
 
 
 def init_app(app):
@@ -412,7 +377,7 @@ def setup_sqlalchemy_events(app):
                         'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
                         'url_rule': current_task.name,  # task name
                     }
-                # anything else. migrations possibly, or flask cli commands.
+                # anything else. migrations possibly.
                 else:
                     current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
                     connection_record.info['request_data'] = {

app/accept_invite/rest.py

@@ -0,0 +1,52 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    current_app
+)
+
+from itsdangerous import SignatureExpired, BadData
+
+from notifications_utils.url_safe_token import check_token
+
+from app.dao.invited_user_dao import get_invited_user_by_id
+from app.dao.organisation_dao import dao_get_invited_organisation_user
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
+from app.schemas import invited_user_schema
+
+
+accept_invite = Blueprint('accept_invite', __name__)
+register_errors(accept_invite)
+
+
+@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
+def validate_invitation_token(invitation_type, token):
+
+    max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
+
+    try:
+        invited_user_id = check_token(token,
+                                      current_app.config['SECRET_KEY'],
+                                      current_app.config['DANGEROUS_SALT'],
+                                      max_age_seconds)
+    except SignatureExpired:
+        errors = {'invitation':
+                  'Your invitation to GOV.UK Notify has expired. '
+                  'Please ask the person that invited you to send you another one'}
+        raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
+
+    if invitation_type == 'service':
+        invited_user = get_invited_user_by_id(invited_user_id)
+        return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+    elif invitation_type == 'organisation':
+        invited_user = dao_get_invited_organisation_user(invited_user_id)
+        return jsonify(data=invited_user.serialize()), 200
+    else:
+        raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))

app/authentication/auth.py

@@ -1,23 +1,16 @@
-import uuid
-
-from flask import current_app, g, request
-from gds_metrics import Histogram
-from notifications_python_client.authentication import (
-    decode_jwt_token,
-    get_token_issuer,
-)
+from flask import request, _request_ctx_stack, current_app, g
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
 from notifications_python_client.errors import (
-    TokenAlgorithmError,
-    TokenDecodeError,
-    TokenError,
-    TokenExpiredError,
-    TokenIssuerError,
+    TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError, TokenError
 )
 from notifications_utils import request_helper
+from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
+from gds_metrics import Histogram
 
 from app.serialised_models import SerialisedService
 
+
 GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header'  # noqa
 
 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
@@ -49,115 +42,7 @@ class AuthError(Exception):
         }
 
 
-class InternalApiKey():
-    def __init__(self, client_id, secret):
-        self.secret = secret
-        self.id = client_id
-        self.expiry_date = None
-
-
-def requires_no_auth():
-    pass
-
-
-def requires_govuk_alerts_auth():
-    requires_internal_auth(current_app.config.get('GOVUK_ALERTS_CLIENT_ID'))
-
-
-def requires_admin_auth():
-    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
-
-
-def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
-        raise TypeError("Unknown client_id for internal auth")
-
-    request_helper.check_proxy_header_before_request()
-    auth_token = _get_auth_token(request)
-    client_id = _get_token_issuer(auth_token)
-
-    if client_id != expected_client_id:
-        current_app.logger.info('client_id: %s', client_id)
-        current_app.logger.info('expected_client_id: %s', expected_client_id)
-        raise AuthError("Unauthorized: not allowed to perform this action", 401)
-
-    api_keys = [
-        InternalApiKey(client_id, secret)
-        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
-    ]
-
-    _decode_jwt_token(auth_token, api_keys, client_id)
-    g.service_id = client_id
-
-
-def requires_auth():
-    request_helper.check_proxy_header_before_request()
-
-    auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
-
-    try:
-        service_id = uuid.UUID(issuer)
-    except Exception:
-        raise AuthError("Invalid token: service id is not the right data type", 403)
-
-    try:
-        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
-            service = SerialisedService.from_id(service_id)
-    except NoResultFound:
-        raise AuthError("Invalid token: service not found", 403)
-
-    if not service.api_keys:
-        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
-
-    if not service.active:
-        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
-
-    api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
-
-    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
-        service_id,
-        api_key.id,
-        request.headers.get('User-Agent'),
-        request.base_url
-    ))
-
-    g.api_user = api_key
-    g.service_id = service_id
-    g.authenticated_service = service
-
-
-def _decode_jwt_token(auth_token, api_keys, service_id=None):
-    for api_key in api_keys:
-        try:
-            decode_jwt_token(auth_token, api_key.secret)
-        except TokenExpiredError:
-            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenAlgorithmError:
-            err_msg = "Invalid token: algorithm used is not HS256"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
-        except TokenDecodeError:
-            # we attempted to validate the token but it failed meaning it was not signed using this api key.
-            # Let's try the next one
-            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
-            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
-            # continue on to check the next API key
-            continue
-        except TokenError:
-            # General error when trying to decode and validate the token
-            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
-
-        if api_key.expiry_date:
-            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
-
-        return api_key
-    else:
-        # service has API keys, but none matching the one the user provided
-        raise AuthError("Invalid token: API key not found", 403, service_id=service_id)
-
-
-def _get_auth_token(req):
+def get_auth_token(req):
     auth_header = req.headers.get('Authorization', None)
     if not auth_header:
         raise AuthError('Unauthorized: authentication token must be provided', 401)
@@ -170,7 +55,97 @@ def _get_auth_token(req):
     return auth_header[7:]
 
 
-def _get_token_issuer(auth_token):
+def requires_no_auth():
+    pass
+
+
+def requires_admin_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
+
+    if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
+        g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
+
+        for secret in current_app.config.get('API_INTERNAL_SECRETS'):
+            try:
+                decode_jwt_token(auth_token, secret)
+                return
+            except TokenExpiredError:
+                raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
+            except TokenDecodeError:
+                # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions
+                # (which are children of `TokenDecodeError`) as these should cause an auth error immediately rather
+                # than continue on to check the next admin client secret
+                continue
+
+        # Either there are no admin client secrets or their token didn't match one of them so error
+        raise AuthError("Unauthorized: admin authentication token not found", 401)
+    else:
+        raise AuthError('Unauthorized: admin authentication token required', 401)
+
+
+def requires_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_token = get_auth_token(request)
+    issuer = __get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
+
+    try:
+        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
+            service = SerialisedService.from_id(issuer)
+    except DataError:
+        raise AuthError("Invalid token: service id is not the right data type", 403)
+    except NoResultFound:
+        raise AuthError("Invalid token: service not found", 403)
+
+    if not service.api_keys:
+        raise AuthError("Invalid token: service has no API keys", 403, service_id=service.id)
+
+    if not service.active:
+        raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
+
+    for api_key in service.api_keys:
+        try:
+            decode_jwt_token(auth_token, api_key.secret)
+        except TokenExpiredError:
+            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenAlgorithmError:
+            err_msg = "Invalid token: algorithm used is not HS256"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
+        except TokenDecodeError:
+            # we attempted to validate the token but it failed meaning it was not signed using this api key.
+            # Let's try the next one
+            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
+            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
+            # continue on to check the next API key
+            continue
+        except TokenError:
+            # General error when trying to decode and validate the token
+            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service.id, api_key_id=api_key.id)
+
+        if api_key.expiry_date:
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
+
+        g.service_id = service.id
+        _request_ctx_stack.top.authenticated_service = service
+        _request_ctx_stack.top.api_user = api_key
+
+        current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
+            service.id,
+            api_key.id,
+            request.headers.get('User-Agent'),
+            request.base_url
+        ))
+        return
+    else:
+        # service has API keys, but none matching the one the user provided
+        raise AuthError("Invalid token: API key not found", 403, service_id=service.id)
+
+
+def __get_token_issuer(auth_token):
     try:
         issuer = get_token_issuer(auth_token)
     except TokenIssuerError:

app/aws/s3.py

@@ -1,7 +1,8 @@
-import botocore
-from boto3 import client, resource
 from flask import current_app
 
+from boto3 import client, resource
+import botocore
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
@@ -69,6 +70,22 @@ def remove_contact_list_from_s3(service_id, contact_list_id):
     return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))
 
 
+def get_s3_bucket_objects(bucket_name, subfolder=''):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        if page.get('Contents'):
+            all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
+
+
 def remove_s3_object(bucket_name, object_key):
     obj = get_s3_object(bucket_name, object_key)
     return obj.delete()

app/billing/billing_schemas.py

@@ -1,45 +1,42 @@
 from datetime import datetime
 
+
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
     "type": "object",
     "title": "Create",
     "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
     },
     "required": ["free_sms_fragment_limit"]
 }
 
 
-def serialize_ft_billing_remove_emails(rows):
-    return [
-        {
-            "month": (datetime.strftime(row.month, "%B")),
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "postage": row.postage,
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_remove_emails(data):
+    results = []
+    billed_notifications = [x for x in data if x.notification_type != 'email']
+    for notification in billed_notifications:
+        json_result = {
+            "month": (datetime.strftime(notification.month, "%B")),
+            "notification_type": notification.notification_type,
+            "billing_units": notification.billable_units,
+            "rate": float(notification.rate),
+            "postage": notification.postage,
         }
-        for row in rows
-        if row.notification_type != 'email'
-    ]
+        results.append(json_result)
+    return results
 
 
-def serialize_ft_billing_yearly_totals(rows):
-    return [
-        {
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
         }
-        for row in rows
-    ]
+        yearly_totals.append(json_result)
+
+    return yearly_totals

app/billing/rest.py

@@ -6,18 +6,18 @@ from app.billing.billing_schemas import (
     serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
     dao_get_free_sms_fragment_limit_for_year,
-    dao_update_annual_billing_for_future_years,
-    set_default_free_allowance_for_service,
+    dao_get_all_free_sms_fragment_limit,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
-    fetch_billing_totals_for_year,
-    fetch_monthly_billing_for_year,
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
 )
+
+from app.errors import InvalidRequest
 from app.errors import register_errors
-from app.models import Service
 from app.schema_validation import validate
 
 billing_blueprint = Blueprint(
@@ -30,6 +30,7 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
+@billing_blueprint.route('/ft-monthly-usage')
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
@@ -41,6 +42,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
+@billing_blueprint.route('/ft-yearly-usage-summary')
 @billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:
@@ -61,14 +63,27 @@ def get_free_sms_fragment_limit(service_id):
     annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
     if annual_billing is None:
-        service = Service.query.get(service_id)
-        # An entry does not exist in annual_billing table for that service and year.
-        # Set the annual billing to the default free allowance based on the organisation type of the service.
+        # An entry does not exist in annual_billing table for that service and year. If it is a past year,
+        # we return the oldest entry.
+        # If it is the current or future years, we create an entry in the db table using the newest record,
+        # and return that number.  If all fails, we return InvalidRequest.
+        sms_list = dao_get_all_free_sms_fragment_limit(service_id)
 
-        annual_billing = set_default_free_allowance_for_service(
-            service=service,
-            year_start=int(financial_year_start) if financial_year_start else None
-        )
+        if not sms_list:
+            raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
+        else:
+            if financial_year_start is None:
+                financial_year_start = get_current_financial_year_start_year()
+
+            if int(financial_year_start) < get_current_financial_year_start_year():
+                # return the earliest historical entry
+                annual_billing = sms_list[0]   # The oldest entry
+            else:
+                annual_billing = sms_list[-1]  # The newest entry
+
+                annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
+                                                                              annual_billing.free_sms_fragment_limit,
+                                                                              financial_year_start)
 
     return jsonify(annual_billing.serialize_free_sms_items()), 200
 

app/broadcast_message/broadcast_message_schema.py

@@ -1,8 +1,8 @@
-from app.models import BroadcastStatusType
 from app.schema_validation.definitions import uuid
+from app.models import BroadcastStatusType
 
 create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
     'description': 'POST create broadcast_message schema',
     'type': 'object',
     'title': 'Create broadcast_message',
@@ -13,26 +13,15 @@ create_broadcast_message_schema = {
         'personalisation': {'type': 'object'},
         'starts_at': {'type': 'string', 'format': 'datetime'},
         'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
-        'content': {'type': 'string', 'minLength': 1},
-        'reference': {'type': 'string', 'minLength': 1, 'maxLength': 255},
+        'areas': {"type": "array", "items": {"type": "string"}},
+        'simple_polygons': {"type": "array", "items": {"type": "array"}},
     },
-    'required': ['service_id', 'created_by'],
-    'allOf': [
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['content']},
-        ]},
-        {'oneOf': [
-            {'required': ['template_id']},
-            {'required': ['reference']},
-        ]},
-    ],
+    'required': ['template_id', 'service_id', 'created_by'],
     'additionalProperties': False
 }
 
 update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
     'description': 'POST update broadcast_message schema',
     'type': 'object',
     'title': 'Update broadcast_message',
@@ -40,14 +29,15 @@ update_broadcast_message_schema = {
         'personalisation': {'type': 'object'},
         'starts_at': {'type': 'string', 'format': 'datetime'},
         'finishes_at': {'type': 'string', 'format': 'datetime'},
-        'areas': {'type': 'object'},
+        'areas': {"type": "array", "items": {"type": "string"}},
+        'simple_polygons': {"type": "array", "items": {"type": "array"}},
     },
     'required': [],
     'additionalProperties': False
 }
 
 update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
+    '$schema': 'http://json-schema.org/draft-04/schema#',
     'description': 'POST update broadcast_message status schema',
     'type': 'object',
     'title': 'Update broadcast_message',

app/broadcast_message/rest.py

@@ -1,23 +1,24 @@
-import iso8601
-from flask import Blueprint, jsonify, request
-from notifications_utils.template import BroadcastMessageTemplate
+from datetime import datetime
 
-from app.broadcast_message import utils as broadcast_utils
+import iso8601
+from flask import Blueprint, jsonify, request, current_app
+from app.config import QueueNames
+from app.dao.dao_utils import dao_save_object
+from app.dao.templates_dao import dao_get_template_by_id_and_service_id
+from app.dao.users_dao import get_user_by_id
+from app.dao.broadcast_message_dao import (
+    dao_get_broadcast_message_by_id_and_service_id,
+    dao_get_broadcast_messages_for_service,
+)
+from app.dao.services_dao import dao_fetch_service_by_id
+from app.errors import register_errors, InvalidRequest
+from app.models import BroadcastMessage, BroadcastStatusType, BroadcastEvent, BroadcastEventMessageType
+from app.celery.broadcast_message_tasks import send_broadcast_event
 from app.broadcast_message.broadcast_message_schema import (
     create_broadcast_message_schema,
     update_broadcast_message_schema,
     update_broadcast_message_status_schema,
 )
-from app.dao.broadcast_message_dao import (
-    dao_get_broadcast_message_by_id_and_service_id,
-    dao_get_broadcast_messages_for_service,
-)
-from app.dao.dao_utils import dao_save_object
-from app.dao.services_dao import dao_fetch_service_by_id
-from app.dao.templates_dao import dao_get_template_by_id_and_service_id
-from app.dao.users_dao import get_user_by_id
-from app.errors import InvalidRequest, register_errors
-from app.models import BroadcastMessage, BroadcastStatusType
 from app.schema_validation import validate
 
 broadcast_message_blueprint = Blueprint(
@@ -34,6 +35,49 @@ def _parse_nullable_datetime(dt):
     return dt
 
 
+def _update_broadcast_message(broadcast_message, new_status, updating_user):
+    if updating_user not in broadcast_message.service.users:
+        raise InvalidRequest(
+            f'User {updating_user.id} cannot approve broadcast_message {broadcast_message.id} from other service',
+            status_code=400
+        )
+
+    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
+        raise InvalidRequest(
+            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
+            status_code=400
+        )
+
+    if new_status == BroadcastStatusType.BROADCASTING:
+        # TODO: Remove this platform admin shortcut when the feature goes live
+        if updating_user == broadcast_message.created_by and not (
+            # platform admins and trial mode services can approve their own broadcasts
+            updating_user.platform_admin or
+            broadcast_message.service.restricted
+        ):
+            raise InvalidRequest(
+                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
+                status_code=400
+            )
+        elif not broadcast_message.areas:
+            raise InvalidRequest(
+                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
+                status_code=400
+            )
+        else:
+            broadcast_message.approved_at = datetime.utcnow()
+            broadcast_message.approved_by = updating_user
+
+    if new_status == BroadcastStatusType.CANCELLED:
+        broadcast_message.cancelled_at = datetime.utcnow()
+        broadcast_message.cancelled_by = updating_user
+
+    current_app.logger.info(
+        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
+    )
+    broadcast_message.status = new_status
+
+
 @broadcast_message_blueprint.route('', methods=['GET'])
 def get_broadcast_messages_for_service(service_id):
     # TODO: should this return template content/data in some way? or can we rely on them being cached admin side.
@@ -55,48 +99,18 @@ def create_broadcast_message(service_id):
     validate(data, create_broadcast_message_schema)
     service = dao_fetch_service_by_id(data['service_id'])
     user = get_user_by_id(data['created_by'])
-    personalisation = data.get('personalisation', {})
-    template_id = data.get('template_id')
-
-    if template_id:
-        template = dao_get_template_by_id_and_service_id(
-            template_id, data['service_id']
-        )
-        content = str(template._as_utils_template_with_personalisation(
-            personalisation
-        ))
-        reference = None
-    else:
-        temporary_template = BroadcastMessageTemplate.from_content(data['content'])
-        if temporary_template.content_too_long:
-            raise InvalidRequest(
-                (
-                    f'Content must be '
-                    f'{temporary_template.max_content_count:,.0f} '
-                    f'characters or fewer'
-                ) + (
-                    ' (because it could not be GSM7 encoded)'
-                    if temporary_template.non_gsm_characters else ''
-                ),
-                status_code=400,
-            )
-        template = None
-        content = str(temporary_template)
-        reference = data['reference']
+    template = dao_get_template_by_id_and_service_id(data['template_id'], data['service_id'])
 
     broadcast_message = BroadcastMessage(
         service_id=service.id,
-        template_id=template_id,
-        template_version=template.version if template else None,
-        personalisation=personalisation,
-        areas=data.get("areas", {}),
+        template_id=template.id,
+        template_version=template.version,
+        personalisation=data.get('personalisation', {}),
+        areas={"areas": data.get("areas", []), "simple_polygons": data.get("simple_polygons", [])},
         status=BroadcastStatusType.DRAFT,
         starts_at=_parse_nullable_datetime(data.get('starts_at')),
         finishes_at=_parse_nullable_datetime(data.get('finishes_at')),
         created_by_id=user.id,
-        content=content,
-        reference=reference,
-        stubbed=service.restricted
     )
 
     dao_save_object(broadcast_message)
@@ -107,6 +121,7 @@ def create_broadcast_message(service_id):
 @broadcast_message_blueprint.route('/<uuid:broadcast_message_id>', methods=['POST'])
 def update_broadcast_message(service_id, broadcast_message_id):
     data = request.get_json()
+
     validate(data, update_broadcast_message_schema)
 
     broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
@@ -117,11 +132,9 @@ def update_broadcast_message(service_id, broadcast_message_id):
             status_code=400
         )
 
-    areas = data.get("areas", {})
-
-    if ('ids' in areas and 'simple_polygons' not in areas) or ('ids' not in areas and 'simple_polygons' in areas):
+    if ('areas' in data and 'simple_polygons' not in data) or ('areas' not in data and 'simple_polygons' in data):
         raise InvalidRequest(
-            f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.',
+            f'Cannot update broadcast_message {broadcast_message.id}, areas or polygons are missing.',
             status_code=400
         )
 
@@ -131,8 +144,8 @@ def update_broadcast_message(service_id, broadcast_message_id):
         broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at'])
     if 'finishes_at' in data:
         broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at'])
-    if 'ids' in areas and 'simple_polygons' in areas:
-        broadcast_message.areas = areas
+    if 'areas' in data and 'simple_polygons' in data:
+        broadcast_message.areas = {"areas": data["areas"], "simple_polygons": data["simple_polygons"]}
 
     dao_save_object(broadcast_message)
 
@@ -146,20 +159,53 @@ def update_broadcast_message_status(service_id, broadcast_message_id):
     validate(data, update_broadcast_message_status_schema)
     broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id)
 
-    if not broadcast_message.service.active:
-        raise InvalidRequest("Updating broadcast message is not allowed: service is inactive ", 403)
-
     new_status = data['status']
     updating_user = get_user_by_id(data['created_by'])
 
-    if updating_user not in broadcast_message.service.users:
-        #  we allow platform admins to cancel broadcasts, and we don't check user if request was done via API
-        if not (new_status == BroadcastStatusType.CANCELLED and updating_user.platform_admin):
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot update broadcast_message {broadcast_message.id} from other service',
-                status_code=400
-            )
+    _update_broadcast_message(broadcast_message, new_status, updating_user)
+    dao_save_object(broadcast_message)
 
-    broadcast_utils.update_broadcast_message_status(broadcast_message, new_status, updating_user)
+    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
+        _create_broadcast_event(broadcast_message)
 
     return jsonify(broadcast_message.serialize()), 200
+
+
+def _create_broadcast_event(broadcast_message):
+    """
+    Creates a broadcast event, stores it in the database, and triggers the task to send the CAP XML off
+    """
+    msg_types = {
+        BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
+        BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
+    }
+
+    if broadcast_message.status == BroadcastStatusType.CANCELLED:
+        transmitted_finishes_at = broadcast_message.cancelled_at
+    else:
+        transmitted_finishes_at = broadcast_message.finishes_at
+
+    # TODO: This doesn't support placeholders yet. We shouldn't use BroadcastMessageTemplate when we add placeholders
+    # as that just outputs XML, we need the raw text.
+    event = BroadcastEvent(
+        service=broadcast_message.service,
+        broadcast_message=broadcast_message,
+        message_type=msg_types[broadcast_message.status],
+        transmitted_content={"body": broadcast_message.template.content},
+        transmitted_areas=broadcast_message.areas,
+        # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it include
+        # a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
+        transmitted_sender='notifications.service.gov.uk',
+
+        # TODO: Should this be set to now? Or the original starts_at?
+        transmitted_starts_at=broadcast_message.starts_at,
+        # TODO: When cancelling, do we need to set this to now? Or should we keep it as the original time.
+        transmitted_finishes_at=transmitted_finishes_at,
+    )
+
+    dao_save_object(event)
+
+    send_broadcast_event.apply_async(
+        kwargs={'broadcast_event_id': str(event.id)},
+        queue=QueueNames.NOTIFY
+    )

app/broadcast_message/translators.py

@@ -1,37 +0,0 @@
-from bs4 import BeautifulSoup
-
-
-def cap_xml_to_dict(cap_xml):
-    # This function assumes that it’s being passed valid CAP XML
-    cap = BeautifulSoup(cap_xml, "xml")
-    return {
-        "msgType": cap.alert.msgType.text,
-        "reference": cap.alert.identifier.text,
-        "references": (
-            # references to previous events belonging to the same alert
-            cap.alert.references.text if cap.alert.references else None
-        ),
-        "cap_event": cap.alert.info.event.text,
-        "category": cap.alert.info.category.text,
-        "expires": cap.alert.info.expires.text,
-        "content": cap.alert.info.description.text,
-        "areas": [
-            {
-                "name": area.areaDesc.text,
-                "polygons": [
-                    cap_xml_polygon_to_list(polygon.text)
-                    for polygon in area.find_all('polygon')
-                ]
-            }
-            for area in cap.alert.info.find_all('area')
-        ]
-    }
-
-
-def cap_xml_polygon_to_list(polygon_string):
-    return [
-        [
-            float(coordinate) for coordinate in pair.split(',')
-        ]
-        for pair in polygon_string.strip().split(' ')
-    ]

app/broadcast_message/utils.py

@@ -1,140 +0,0 @@
-import inspect
-from datetime import datetime
-
-from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-
-from app import zendesk_client
-from app.celery.broadcast_message_tasks import send_broadcast_event
-from app.config import QueueNames
-from app.dao.dao_utils import dao_save_object
-from app.errors import InvalidRequest
-from app.models import (
-    BroadcastEvent,
-    BroadcastEventMessageType,
-    BroadcastStatusType,
-)
-
-
-def update_broadcast_message_status(broadcast_message, new_status, updating_user=None, api_key_id=None):
-    _validate_broadcast_update(broadcast_message, new_status, updating_user)
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        broadcast_message.approved_at = datetime.utcnow()
-        broadcast_message.approved_by = updating_user
-
-    if new_status == BroadcastStatusType.CANCELLED:
-        broadcast_message.cancelled_at = datetime.utcnow()
-        broadcast_message.cancelled_by = updating_user
-        broadcast_message.cancelled_by_api_key_id = api_key_id
-
-    current_app.logger.info(
-        f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}'
-    )
-    broadcast_message.status = new_status
-
-    dao_save_object(broadcast_message)
-    _create_p1_zendesk_alert(broadcast_message)
-
-    if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}:
-        _create_broadcast_event(broadcast_message)
-
-
-def _validate_broadcast_update(broadcast_message, new_status, updating_user):
-    if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]:
-        raise InvalidRequest(
-            f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}',
-            status_code=400
-        )
-
-    if new_status == BroadcastStatusType.BROADCASTING:
-        # training mode services can approve their own broadcasts
-        if updating_user == broadcast_message.created_by and not broadcast_message.service.restricted:
-            raise InvalidRequest(
-                f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}',
-                status_code=400
-            )
-        elif len(broadcast_message.areas['simple_polygons']) == 0:
-            raise InvalidRequest(
-                f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.',
-                status_code=400
-            )
-
-
-def _create_p1_zendesk_alert(broadcast_message):
-    if current_app.config['NOTIFY_ENVIRONMENT'] != 'live':
-        return
-
-    if broadcast_message.status != BroadcastStatusType.BROADCASTING:
-        return
-
-    message = inspect.cleandoc(f"""
-        Broadcast Sent
-
-        https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id}
-
-        Sent on channel {broadcast_message.service.broadcast_channel} to {broadcast_message.areas["names"]}.
-
-        Content starts "{broadcast_message.content[:100]}".
-
-        Follow the runbook to check the broadcast went out OK:
-        https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs/edit#heading=h.lzr9aq5b4wg
-    """)
-
-    ticket = NotifySupportTicket(
-        subject='Live broadcast sent',
-        message=message,
-        ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-        technical_ticket=True,
-        org_id=current_app.config['BROADCAST_ORGANISATION_ID'],
-        org_type='central',
-        service_id=str(broadcast_message.service_id),
-        p1=True
-    )
-    zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-def _create_broadcast_event(broadcast_message):
-    """
-    If the service is live and the broadcast message is not stubbed, creates a broadcast event, stores it in the
-    database, and triggers the task to send the CAP XML off.
-    """
-    service = broadcast_message.service
-
-    if not broadcast_message.stubbed and not service.restricted:
-        msg_types = {
-            BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT,
-            BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL,
-        }
-
-        event = BroadcastEvent(
-            service=service,
-            broadcast_message=broadcast_message,
-            message_type=msg_types[broadcast_message.status],
-            transmitted_content={"body": broadcast_message.content},
-            transmitted_areas=broadcast_message.areas,
-            # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it
-            # include a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar
-            transmitted_sender='notifications.service.gov.uk',
-
-            # TODO: Should this be set to now? Or the original starts_at?
-            transmitted_starts_at=broadcast_message.starts_at,
-            transmitted_finishes_at=broadcast_message.finishes_at,
-        )
-
-        dao_save_object(event)
-
-        send_broadcast_event.apply_async(
-            kwargs={'broadcast_event_id': str(event.id)},
-            queue=QueueNames.BROADCASTS
-        )
-    elif broadcast_message.stubbed != service.restricted:
-        # It's possible for a service to create a broadcast in trial mode, and then approve it after the
-        # service is live (or vice versa). We don't think it's safe to send such broadcasts, as the service
-        # has changed since they were created. Log an error instead.
-        current_app.logger.error(
-            f'Broadcast event not created. Stubbed status of broadcast message was {broadcast_message.stubbed}'
-            f' but service was {"in trial mode" if service.restricted else "live"}'
-        )

app/celery/broadcast_message_tasks.py

@@ -1,230 +1,31 @@
-from datetime import datetime
-
+import requests
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 
-from app import cbc_proxy_client, notify_celery
-from app.clients.cbc_proxy import CBCProxyRetryableException
-from app.config import QueueNames, TaskNames
-from app.dao.broadcast_message_dao import (
-    create_broadcast_provider_message,
-    dao_get_broadcast_event_by_id,
-    update_broadcast_provider_message_status,
-)
-from app.models import (
-    BroadcastEventMessageType,
-    BroadcastProvider,
-    BroadcastProviderMessageStatus,
-)
-from app.utils import format_sequential_number
+from app import notify_celery
 
-
-class BroadcastIntegrityError(Exception):
-    pass
-
-
-def get_retry_delay(retry_count):
-    """
-    Given a count of retries so far, return a delay for the next one.
-    `retry_count` should be 0 the first time a task fails.
-    """
-    # TODO: replace with celery's built in exponential backoff
-
-    # 2 to the power of x. 1, 2, 4, 8, 16, 32, ...
-    delay = 2**retry_count
-    # never wait longer than 4 minutes
-    return min(delay, 240)
-
-
-def check_event_is_authorised_to_be_sent(broadcast_event, provider):
-    if not broadcast_event.service.active:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is suspended'
-        )
-
-    if broadcast_event.service.restricted:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the service is not live'
-        )
-
-    if broadcast_event.broadcast_message.stubbed:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: the broadcast message is stubbed'
-        )
-
-
-def check_event_makes_sense_in_sequence(broadcast_event, provider):
-    """
-    If any previous event hasn't sent yet for that provider, then we shouldn't send the current event. Instead, fail and
-    raise a zendesk ticket - so that a notify team member can assess the state of the previous messages, and if
-    necessary, can replay the `send_broadcast_provider_message` task if the previous message has now been sent.
-
-    Note: This is called before the new broadcast_provider_message is created.
-
-    # Help, I've come across this code following a pagerduty alert, what should I do?
-
-    1. Find the failing broadcast_provider_message associated with the previous event that caused this to trip.
-    2. If that provider message is still failing to send, fix the issue causing that. The task to send that previous
-       message might still be retrying in the background - look for logs related to that task.
-    3. If that provider message has sent succesfully, you might need to send this task off depending on context. This
-       might not always be true though, for example, it may not be necessary to send a cancel if the original alert has
-       already expired.
-    4. If you need to re-send this task off again, you'll need to run the following command on paas:
-       `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)`
-    """
-    current_provider_message = broadcast_event.get_provider_message(provider)
-    # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING:
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'It is in status {current_provider_message.status}'
-        )
-
-    if broadcast_event.transmitted_finishes_at < datetime.utcnow():
-        raise BroadcastIntegrityError(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'The expiry time of {broadcast_event.transmitted_finishes_at} has already passed'
-        )
-
-    # get events sorted from earliest to latest
-    events = sorted(broadcast_event.broadcast_message.events, key=lambda x: x.sent_at)
-
-    for prev_event in events:
-        if prev_event.id != broadcast_event.id and prev_event.sent_at < broadcast_event.sent_at:
-            # get the record from when that event was sent to the same provider
-            prev_provider_message = prev_event.get_provider_message(provider)
-
-            # the previous message hasn't even got round to running `send_broadcast_provider_message` yet.
-            if not prev_provider_message:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has no provider_message for provider {provider} yet.\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
-
-            # if there's a previous message that has started but not finished sending (whether it fatally errored or is
-            # currently retrying)
-            if prev_provider_message.status != BroadcastProviderMessageStatus.ACK:
-                raise BroadcastIntegrityError(
-                    f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' +
-                    f'(type {prev_event.message_type}) has not finished sending to provider {provider} yet.\n' +
-                    f'It is currently in status "{prev_provider_message.status}".\n' +
-                    'You must ensure that the other event sends succesfully, then manually kick off this event ' +
-                    'again by re-running send_broadcast_provider_message for this event and provider.'
-                )
+from app.dao.broadcast_message_dao import dao_get_broadcast_event_by_id
 
 
 @notify_celery.task(name="send-broadcast-event")
-def send_broadcast_event(broadcast_event_id):
+@statsd(namespace="tasks")
+def send_broadcast_event(broadcast_event_id, provider='stub-1'):
     broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
 
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )
-
-    for provider in broadcast_event.service.get_available_broadcast_providers():
-        send_broadcast_provider_message.apply_async(
-            kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider},
-            queue=QueueNames.BROADCASTS
-        )
-
-
-# max_retries=None: retry forever
-@notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None)
-def send_broadcast_provider_message(self, broadcast_event_id, provider):
-    if not current_app.config['CBC_PROXY_ENABLED']:
-        current_app.logger.info(
-            "CBC Proxy disabled, not sending broadcast_provider_message for "
-            f"broadcast_event_id {broadcast_event_id} with provider {provider}"
-        )
-        return
-
-    broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id)
-
-    check_event_is_authorised_to_be_sent(broadcast_event, provider)
-    check_event_makes_sense_in_sequence(broadcast_event, provider)
-
-    # the broadcast_provider_message may already exist if we retried previously
-    broadcast_provider_message = broadcast_event.get_provider_message(provider)
-    if broadcast_provider_message is None:
-        broadcast_provider_message = create_broadcast_provider_message(broadcast_event, provider)
-
-    formatted_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        formatted_message_number = format_sequential_number(broadcast_provider_message.message_number)
-
     current_app.logger.info(
-        f'Invoking cbc proxy to send broadcast_provider_message with ID of {broadcast_provider_message.id} '
-        f'and broadcast_event ID of {broadcast_event_id} '
-        f'msgType {broadcast_event.message_type}'
+        f'sending broadcast_event {broadcast_event.reference} '
+        f'msgType {broadcast_event.message_type} to {provider}'
     )
 
-    areas = [
-        {"polygon": polygon}
-        for polygon in broadcast_event.transmitted_areas["simple_polygons"]
-    ]
+    payload = broadcast_event.serialize()
 
-    cbc_proxy_provider_client = cbc_proxy_client.get_proxy(provider)
+    resp = requests.post(
+        f'{current_app.config["CBC_PROXY_URL"]}/broadcasts/events/{provider}',
+        json=payload
+    )
+    resp.raise_for_status()
 
-    try:
-        if broadcast_event.message_type == BroadcastEventMessageType.ALERT:
-            cbc_proxy_provider_client.create_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.UPDATE:
-            cbc_proxy_provider_client.update_and_send_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                headline="GOV.UK Notify Broadcast",
-                description=broadcast_event.transmitted_content['body'],
-                areas=areas,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-                expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string,
-                # We think an alert update should always go out on the same channel that created the alert
-                # We recognise there is a small risk with this code here that if the services channel was
-                # changed between an alert being sent out and then updated, then something might go wrong
-                # but we are relying on service channels changing almost never, and not mid incident
-                # We may consider in the future, changing this such that we store the channel a broadcast was
-                # sent on on the broadcast message itself and pick the value from there instead of the service
-                channel=broadcast_event.service.broadcast_channel
-            )
-        elif broadcast_event.message_type == BroadcastEventMessageType.CANCEL:
-            cbc_proxy_provider_client.cancel_broadcast(
-                identifier=str(broadcast_provider_message.id),
-                message_number=formatted_message_number,
-                previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider),
-                sent=broadcast_event.sent_at_as_cap_datetime_string,
-            )
-    except CBCProxyRetryableException as exc:
-        delay = get_retry_delay(self.request.retries)
-        current_app.logger.exception(
-            f'Retrying send_broadcast_provider_message for broadcast event {broadcast_event_id}, '
-            f'provider message {broadcast_provider_message.id}, provider {provider} in {delay} seconds'
-        )
-
-        self.retry(
-            exc=exc,
-            countdown=delay,
-            queue=QueueNames.BROADCASTS,
-        )
-
-    update_broadcast_provider_message_status(broadcast_provider_message, status=BroadcastProviderMessageStatus.ACK)
-
-
-@notify_celery.task(name='trigger-link-test')
-def trigger_link_test(provider):
-    cbc_proxy_client.get_proxy(provider).send_link_test()
+    current_app.logger.info(
+        f'broadcast_event {broadcast_event.reference} '
+        f'msgType {broadcast_event.message_type} sent to {provider}'
+    )

app/celery/celery.py

@@ -0,0 +1,78 @@
+import time
+
+from gds_metrics.metrics import Histogram
+from celery import Celery, Task
+from celery.signals import worker_process_shutdown
+from flask import g, request
+from flask.ctx import has_request_context
+
+
+@worker_process_shutdown.connect
+def log_on_worker_shutdown(sender, signal, pid, exitcode, **kwargs):
+    # imported here to avoid circular imports
+    from app import notify_celery
+
+    # if the worker has already restarted at least once, then we no longer have app context and current_app won't work
+    # to create a new one. Instead we have to create a new app context from the original flask app and use that instead.
+    with notify_celery._app.app_context():
+        # if the worker has restarted
+        notify_celery._app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
+
+
+def make_task(app):
+    SQS_APPLY_ASYNC_DURATION_SECONDS = Histogram(
+        'sqs_apply_async_duration_seconds',
+        'Time taken to put task on queue',
+        ['task_name']
+    )
+
+    class NotifyTask(Task):
+        abstract = True
+        start = None
+
+        def on_success(self, retval, task_id, args, kwargs):
+            elapsed_time = time.time() - self.start
+            app.logger.info(
+                "{task_name} took {time}".format(
+                    task_name=self.name, time="{0:.4f}".format(elapsed_time)
+                )
+            )
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            # ensure task will log exceptions to correct handlers
+            app.logger.exception('Celery task: {} failed'.format(self.name))
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def __call__(self, *args, **kwargs):
+            # ensure task has flask context to access config, logger, etc
+            with app.app_context():
+                self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
+                return super().__call__(*args, **kwargs)
+
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            with SQS_APPLY_ASYNC_DURATION_SECONDS.labels(self.name).time():
+                return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
+    return NotifyTask
+
+
+class NotifyCelery(Celery):
+
+    def init_app(self, app):
+        super().__init__(
+            app.import_name,
+            broker=app.config['BROKER_URL'],
+            task_cls=make_task(app),
+        )
+
+        self.conf.update(app.config)
+        self._app = app

app/celery/letters_pdf_tasks.py

@@ -1,47 +1,41 @@
-from base64 import urlsafe_b64encode
 from datetime import datetime, timedelta
 from hashlib import sha512
+from base64 import urlsafe_b64encode
 
 from botocore.exceptions import ClientError as BotoClientError
 from flask import current_app
-from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.postal_address import PostalAddress
+
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import encryption, notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
-from app.cronitor import cronitor
 from app.dao.notifications_dao import (
-    dao_get_letters_and_sheets_volume_by_postage,
-    dao_get_letters_to_be_printed,
-    dao_get_notification_by_reference,
-    dao_update_notification,
-    dao_update_notifications_by_reference,
     get_notification_by_id,
     update_notification_status_by_id,
+    dao_update_notification,
+    dao_get_notification_by_reference,
+    dao_update_notifications_by_reference,
+    dao_get_letters_to_be_printed,
 )
-from app.dao.templates_dao import dao_get_template_by_id
+from app.letters.utils import get_letter_pdf_filename
 from app.errors import VirusScanError
 from app.exceptions import NotificationTechnicalFailureException
 from app.letters.utils import (
-    LetterPDFNotFound,
-    ScanErrorType,
-    find_letter_pdf_in_s3,
-    generate_letter_pdf_filename,
     get_billable_units_for_letter_page_count,
-    get_file_names_from_error_bucket,
-    get_folder_name,
     get_reference_from_filename,
-    move_error_pdf_to_scan_bucket,
+    ScanErrorType,
     move_failed_pdf,
     move_sanitised_letter_to_test_or_live_pdf_bucket,
     move_scan_to_invalid_pdf_bucket,
+    move_error_pdf_to_scan_bucket,
+    get_file_names_from_error_bucket,
 )
 from app.models import (
     INTERNATIONAL_LETTERS,
-    INTERNATIONAL_POSTAGE_TYPES,
-    KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
@@ -51,16 +45,24 @@ from app.models import (
     NOTIFICATION_VIRUS_SCAN_FAILED,
     POSTAGE_TYPES,
     RESOLVE_POSTAGE_FOR_FILE_NAME,
-    Service,
-)
+    INTERNATIONAL_POSTAGE_TYPES)
+from app.cronitor import cronitor
+
+
+@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def create_letters_pdf(self, notification_id):
+    get_pdf_for_templated_letter(notification_id)
 
 
 @notify_celery.task(bind=True, name="get-pdf-for-templated-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def get_pdf_for_templated_letter(self, notification_id):
     try:
         notification = get_notification_by_id(notification_id, _raise=True)
-        letter_filename = generate_letter_pdf_filename(
+        letter_filename = get_letter_pdf_filename(
             reference=notification.reference,
+            crown=notification.service.crown,
             created_at=notification.created_at,
             ignore_folder=notification.key_type == KEY_TYPE_TEST,
             postage=notification.postage
@@ -86,12 +88,12 @@ def get_pdf_for_templated_letter(self, notification_id):
             args=(encrypted_data,),
             queue=QueueNames.SANITISE_LETTERS
         )
-    except Exception as e:
+    except Exception:
         try:
             current_app.logger.exception(
                 f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
             )
-            self.retry(exc=e, queue=QueueNames.RETRY)
+            self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
             message = f"RETRY FAILED: Max retries reached. " \
                       f"The task create-letter-pdf failed for notification id {notification_id}. " \
@@ -101,6 +103,7 @@ def get_pdf_for_templated_letter(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
 def update_billable_units_for_letter(self, notification_id, page_count):
     notification = get_notification_by_id(notification_id, _raise=True)
 
@@ -116,16 +119,6 @@ def update_billable_units_for_letter(self, notification_id, page_count):
         )
 
 
-@notify_celery.task(
-    bind=True, name="update-validation-failed-for-templated-letter", max_retries=15, default_retry_delay=300
-)
-def update_validation_failed_for_templated_letter(self, notification_id, page_count):
-    notification = get_notification_by_id(notification_id, _raise=True)
-    notification.status = NOTIFICATION_VALIDATION_FAILED
-    dao_update_notification(notification)
-    current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}")
-
-
 @notify_celery.task(name='collate-letter-pdfs-to-be-sent')
 @cronitor("collate-letter-pdfs-to-be-sent")
 def collate_letter_pdfs_to_be_sent():
@@ -136,6 +129,7 @@ def collate_letter_pdfs_to_be_sent():
     that have not yet been sent.
     If run after midnight, it will collect up letters created before 5:30pm the day before.
     """
+    current_app.logger.info("starting collate-letter-pdfs-to-be-sent")
     print_run_date = convert_utc_to_bst(datetime.utcnow())
     if print_run_date.time() < LETTER_PROCESSING_DEADLINE:
         print_run_date = print_run_date - timedelta(days=1)
@@ -143,8 +137,6 @@ def collate_letter_pdfs_to_be_sent():
     print_run_deadline = print_run_date.replace(
         hour=17, minute=30, second=0, microsecond=0
     )
-    _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)
-
     for postage in POSTAGE_TYPES:
         current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
         letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
@@ -152,18 +144,13 @@ def collate_letter_pdfs_to_be_sent():
         for i, letters in enumerate(group_letters(letters_to_print)):
             filenames = [letter['Key'] for letter in letters]
 
-            service_id = letters[0]['ServiceId']
-            organisation_id = letters[0]['OrganisationId']
-
             hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
             # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format(
+            dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format(
                 date=print_run_deadline.strftime("%Y-%m-%d"),
                 postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
                 num=i + 1,
-                hash=hash,
-                service_id=service_id,
-                organisation_id=organisation_id
+                hash=hash
             )
 
             current_app.logger.info(
@@ -187,74 +174,25 @@ def collate_letter_pdfs_to_be_sent():
     current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
 
 
-def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
-    letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
-    send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date())
-
-
-def send_letters_volume_email_to_dvla(letters_volumes, date):
-    personalisation = {
-        'total_volume': 0,
-        'first_class_volume': 0,
-        'second_class_volume': 0,
-        'international_volume': 0,
-        'total_sheets': 0,
-        'first_class_sheets': 0,
-        "second_class_sheets": 0,
-        'international_sheets': 0,
-        'date': date.strftime("%d %B %Y")
-    }
-    for item in letters_volumes:
-        personalisation['total_volume'] += item.letters_count
-        personalisation['total_sheets'] += item.sheets_count
-        if f"{item.postage}_class_volume" in personalisation:
-            personalisation[f"{item.postage}_class_volume"] = item.letters_count
-            personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
-        else:
-            personalisation["international_volume"] += item.letters_count
-            personalisation["international_sheets"] += item.sheets_count
-
-    template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
-    recipients = current_app.config['DVLA_EMAIL_ADDRESSES']
-    reply_to = template.service.get_default_reply_to_email_address()
-    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
-
-    # avoid circular imports:
-    from app.notifications.process_notifications import (
-        persist_notification,
-        send_notification_to_queue,
-    )
-    for recipient in recipients:
-        saved_notification = persist_notification(
-            template_id=template.id,
-            template_version=template.version,
-            recipient=recipient,
-            service=service,
-            personalisation=personalisation,
-            notification_type=template.template_type,
-            api_key_id=None,
-            key_type=KEY_TYPE_NORMAL,
-            reply_to_text=reply_to
-        )
-
-        send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
-
-
 def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
     letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
+    letter_pdfs = []
     for letter in letters_awaiting_sending:
         try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
-            yield {
-                "Key": letter_pdf.key,
-                "Size": letter_pdf.size,
-                "ServiceId": str(letter.service_id),
-                "OrganisationId": str(letter.service.organisation_id)
-            }
-        except (BotoClientError, LetterPDFNotFound) as e:
+            letter_file_name = get_letter_pdf_filename(
+                reference=letter.reference,
+                crown=letter.service.crown,
+                created_at=letter.created_at,
+                postage=letter.postage
+            )
+            letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
+            letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
+        except BotoClientError as e:
             current_app.logger.exception(
                 f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
 
+    return letter_pdfs
+
 
 def group_letters(letter_pdfs):
     """
@@ -264,23 +202,16 @@ def group_letters(letter_pdfs):
     """
     running_filesize = 0
     list_of_files = []
-    service_id = None
     for letter in letter_pdfs:
         if letter['Key'].lower().endswith('.pdf'):
-            if not service_id:
-                service_id = letter['ServiceId']
             if (
-                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']
-                or len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
-                or letter['ServiceId'] != service_id
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
             ):
                 yield list_of_files
                 running_filesize = 0
                 list_of_files = []
-                service_id = None
 
-            if not service_id:
-                service_id = letter['ServiceId']
             running_filesize += letter['Size']
             list_of_files.append(letter)
 
@@ -377,8 +308,9 @@ def process_sanitised_letter(self, sanitise_data):
 
         # The original filename could be wrong because we didn't know the postage.
         # Now we know if the letter is international, we can check what the filename should be.
-        upload_file_name = generate_letter_pdf_filename(
+        upload_file_name = get_letter_pdf_filename(
             reference=notification.reference,
+            crown=notification.service.crown,
             created_at=notification.created_at,
             ignore_folder=True,
             postage=notification.postage
@@ -489,7 +421,6 @@ def update_letter_pdf_status(reference, status, billable_units, recipient_addres
         update_dict.update({'postage': postage, 'international': True})
     if recipient_address:
         update_dict['to'] = recipient_address
-        update_dict['normalised_to'] = ''.join(recipient_address.split()).lower()
     return dao_update_notifications_by_reference(
         references=[reference],
         update_dict=update_dict)[0]
@@ -534,37 +465,3 @@ def replay_letters_in_error(filename=None):
                     [filename],
                     queue=QueueNames.LETTERS
                 )
-
-
-@notify_celery.task(name='resanitise-pdf')
-def resanitise_pdf(notification_id):
-    """
-    `notification_id` is the notification id for a PDF letter which was either uploaded or sent using the API.
-
-    This task calls the `recreate_pdf_for_precompiled_letter` template preview task which recreates the
-    PDF for a letter which is already sanitised and in the letters-pdf bucket. The new file that is generated
-    will then overwrite the existing letter in the letters-pdf bucket.
-    """
-    notification = get_notification_by_id(notification_id)
-
-    # folder_name is the folder that the letter is in the letters-pdf bucket e.g. '2021-10-10/'
-    folder_name = get_folder_name(notification.created_at)
-
-    filename = generate_letter_pdf_filename(
-            reference=notification.reference,
-            created_at=notification.created_at,
-            ignore_folder=True,
-            postage=notification.postage
-        )
-
-    notify_celery.send_task(
-        name=TaskNames.RECREATE_PDF_FOR_PRECOMPILED_LETTER,
-        kwargs={
-            'notification_id': str(notification.id),
-            'file_location': f'{folder_name}{filename}',
-            'allow_international_letters': notification.service.has_permission(
-                INTERNATIONAL_LETTERS
-            ),
-        },
-        queue=QueueNames.SANITISE_LETTERS,
-    )

app/celery/nightly_tasks.py

@@ -1,56 +1,55 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
 import pytz
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from notifications_utils.timezones import convert_utc_to_bst
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, statsd_client, zendesk_client
+from app import notify_celery, performance_platform_client, zendesk_client
 from app.aws import s3
+from app.celery.service_callback_tasks import (
+    send_delivery_status_to_service,
+    create_delivery_status_callback_data,
+)
 from app.config import QueueNames
-from app.cronitor import cronitor
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
-    dao_archive_job,
     dao_get_jobs_older_than_data_retention,
+    dao_archive_job
 )
 from app.dao.notifications_dao import (
-    dao_get_notifications_processing_time_stats,
     dao_timeout_notifications,
-    get_service_ids_with_notifications_before,
-    move_notifications_to_notification_history,
-)
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_for_all_services_by_notification_type,
+    delete_notifications_older_than_retention_by_type,
 )
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_NORMAL,
-    LETTER_TYPE,
-    NOTIFICATION_SENDING,
-    SMS_TYPE,
-    FactProcessingTime,
     Notification,
+    NOTIFICATION_SENDING,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    KEY_TYPE_NORMAL
 )
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
+from app.performance_platform import total_sent_notifications, processing_time
+from app.cronitor import cronitor
 from app.utils import get_london_midnight_in_utc
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
+@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
     _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
 
 
 @notify_celery.task(name="remove_letter_jobs")
 @cronitor("remove_letter_jobs")
+@statsd(namespace="tasks")
 def remove_letter_csv_files():
     _remove_csv_files([LETTER_TYPE])
 
@@ -64,111 +63,149 @@ def _remove_csv_files(job_types):
 
 
 @notify_celery.task(name="delete-notifications-older-than-retention")
+@statsd(namespace="tasks")
 def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
+    delete_email_notifications_older_than_retention()
+    delete_sms_notifications_older_than_retention()
+    delete_letter_notifications_older_than_retention()
 
 
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
+@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('sms')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('sms')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise
 
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
+@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('email')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('email')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete email notifications")
+        raise
 
 
 @notify_celery.task(name="delete-letter-notifications")
 @cronitor("delete-letter-notifications")
+@statsd(namespace="tasks")
 def delete_letter_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('letter')
-
-
-def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
-
-    for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_london_midnight_in_utc(
-            convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
-        )
-
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': f.service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': day_to_delete_backwards_from
-        })
-
-    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7))
-    service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
-
-    # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
-    # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
-        notification_type,
-        seven_days_ago
-    )
-
-    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
-
-    for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': seven_days_ago
-        })
-
-    current_app.logger.info(
-        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
-        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
-        f'{len(service_ids_to_purge)} services without flexible data retention'
-    )
-
-
-@notify_celery.task(name='delete-notifications-for-service-and-type')
-def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
-    start = datetime.utcnow()
-    num_deleted = move_notifications_to_notification_history(
-        notification_type,
-        service_id,
-        datetime_to_delete_before,
-    )
-    if num_deleted:
-        end = datetime.utcnow()
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('letter')
         current_app.logger.info(
-            f'delete-notifications-for-service-and-type: '
-            f'service: {service_id}, '
-            f'notification_type: {notification_type}, '
-            f'count deleted: {num_deleted}, '
-            f'duration: {(end - start).seconds} seconds'
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
+                'letter',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
         )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete letter notifications")
+        raise
 
 
 @notify_celery.task(name='timeout-sending-notifications')
 @cronitor('timeout-sending-notifications')
+@statsd(namespace="tasks")
 def timeout_notifications():
-    notifications = ['dummy value so len() > 0']
+    technical_failure_notifications, temporary_failure_notifications = \
+        dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
 
-    cutoff_time = datetime.utcnow() - timedelta(
-        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
+    notifications = technical_failure_notifications + temporary_failure_notifications
+    for notification in notifications:
+        # queue callback task only if the service_callback_api exists
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
+
+    current_app.logger.info(
+        "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    if technical_failure_notifications:
+        message = "{} notifications have been updated to technical-failure because they " \
+                  "have timed out and are still in created.Notification ids: {}".format(
+                      len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
+        raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@cronitor('send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_platform_stats(date=None):
+    # date is a string in the format of "YYYY-MM-DD"
+    if date is None:
+        date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        date = datetime.strptime(date, "%Y-%m-%d").date()
+
+    if performance_platform_client.active:
+
+        send_total_sent_notifications_to_performance_platform(bst_date=date)
+        processing_time.send_processing_time_to_performance_platform(bst_date=date)
+
+
+def send_total_sent_notifications_to_performance_platform(bst_date):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
+    start_time = get_london_midnight_in_utc(bst_date)
+
+    email_sent_count = count_dict['email']
+    sms_sent_count = count_dict['sms']
+    letter_sent_count = count_dict['letter']
+
+    current_app.logger.info(
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
     )
 
-    while len(notifications) > 0:
-        notifications = dao_timeout_notifications(cutoff_time)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'sms',
+        sms_sent_count
+    )
 
-        for notification in notifications:
-            statsd_client.incr(f'timeout-sending.{notification.sent_by}')
-            check_and_queue_callback_task(notification)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'email',
+        email_sent_count
+    )
 
-        current_app.logger.info(
-            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'letter',
+        letter_sent_count
+    )
 
 
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
+@statsd(namespace="tasks")
 def delete_inbound_sms():
     try:
         start = datetime.utcnow()
@@ -187,6 +224,7 @@ def delete_inbound_sms():
 
 @notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
 @cronitor("raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
 def raise_alert_if_letter_notifications_still_sending():
     still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
 
@@ -198,16 +236,11 @@ def raise_alert_if_letter_notifications_still_sending():
         # Only send alerts in production
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"  # noqa
-
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
-                email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
         else:
             current_app.logger.info(message)
 
@@ -233,11 +266,17 @@ def get_letter_notifications_still_sending_when_they_shouldnt_be():
         func.date(Notification.sent_at) <= expected_sent_date
     )
 
+    if today.isoweekday() in {2, 4}:  # on tue, thu, we only care about first class letters
+        q = q.filter(
+            Notification.postage == 'first'
+        )
+
     return q.count(), expected_sent_date
 
 
 @notify_celery.task(name='raise-alert-if-no-letter-ack-file')
 @cronitor('raise-alert-if-no-letter-ack-file')
+@statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
     # get a list of zip files since yesterday
     zip_file_set = set()
@@ -255,57 +294,33 @@ def letter_raise_alert_if_no_ack_file_for_zip():
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))  # noqa
-
-    message = '\n'.join([
-        "Letter ack file does not contain all zip files sent."
-        "",
-        f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n",  # noqa
-        f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent",  # noqa
-        f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
-        "",
-        f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
-    ])
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
 
+    message = (
+        "Letter ack file does not contain all zip files sent. "
+        "Missing ack for zip files: {}, "
+        "pdf bucket: {}, subfolder: {}, "
+        "ack bucket: {}"
+    ).format(
+        str(sorted(zip_file_set - ack_file_set)),
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
+        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
+    )
     # strip empty element before comparison
     ack_file_set.discard('')
     zip_file_set.discard('')
 
     if len(zip_file_set - ack_file_set) > 0:
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
+            zendesk_client.create_ticket(
                 subject="Letter acknowledge error",
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
         current_app.logger.error(message)
 
     if len(ack_file_set - zip_file_set) > 0:
         current_app.logger.info(
             "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
         )
-
-
-@notify_celery.task(name='save-daily-notification-processing-time')
-@cronitor("save-daily-notification-processing-time")
-def save_daily_notification_processing_time(bst_date=None):
-    # bst_date is a string in the format of "YYYY-MM-DD"
-    if bst_date is None:
-        # if a date is not provided, we run against yesterdays data
-        bst_date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        bst_date = datetime.strptime(bst_date, "%Y-%m-%d").date()
-
-    start_time = get_london_midnight_in_utc(bst_date)
-    end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
-    result = dao_get_notifications_processing_time_stats(start_time, end_time)
-    insert_update_processing_time(
-        FactProcessingTime(
-            bst_date=bst_date,
-            messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs
-        )
-    )

app/celery/process_ses_receipts_tasks.py

@@ -3,30 +3,32 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
-from app.clients.email.aws_ses import get_aws_responses
 from app.config import QueueNames
+from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
+
 from app.notifications.notifications_ses_callback import (
-    _check_and_queue_complaint_callback_task,
-    check_and_queue_callback_task,
     determine_notification_bounce_type,
     handle_complaint,
+    _check_and_queue_complaint_callback_task,
+    _check_and_queue_callback_task,
 )
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_ses_results(self, response):
     try:
         ses_message = json.loads(response['Message'])
         notification_type = ses_message['notificationType']
-        bounce_message = None
 
         if notification_type == 'Bounce':
-            notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
@@ -52,9 +54,6 @@ def process_ses_results(self, response):
                 )
             return
 
-        if bounce_message:
-            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
-
         if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
             notifications_dao._duplicate_update_warning(
                 notification=notification,
@@ -70,13 +69,9 @@ def process_ses_results(self, response):
         statsd_client.incr('callback.ses.{}'.format(notification_status))
 
         if notification.sent_at:
-            statsd_client.timing_with_dates(
-                f'callback.ses.{notification_status}.elapsed-time',
-                datetime.utcnow(),
-                notification.sent_at
-            )
+            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
 
-        check_and_queue_callback_task(notification)
+        _check_and_queue_callback_task(notification)
 
         return True
 

app/celery/process_sms_client_response_tasks.py

@@ -2,55 +2,59 @@ import uuid
 from datetime import datetime
 
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
 from app.clients import ClientException
+from app.clients.sms.firetext import get_firetext_responses
+from app.clients.sms.mmg import get_mmg_responses
+from app.celery.service_callback_tasks import send_delivery_status_to_service, create_delivery_status_callback_data
+from app.config import QueueNames
 from app.dao import notifications_dao
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 
-# sms_response_mapper = {
-#     'MMG': get_mmg_responses,
-#     'Firetext': get_firetext_responses,
-# }
+sms_response_mapper = {
+    'MMG': get_mmg_responses,
+    'Firetext': get_firetext_responses
+}
 
 
-# @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
-# def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
-#     # validate reference
-#     try:
-#         uuid.UUID(provider_reference, version=4)
-#     except ValueError as e:
-#         current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
-#         raise e
+@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
+def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
+    # validate reference
+    try:
+        uuid.UUID(provider_reference, version=4)
+    except ValueError as e:
+        current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
+        raise e
 
-#     response_parser = sms_response_mapper[client_name]
+    response_parser = sms_response_mapper[client_name]
 
-#     # validate status
-#     try:
-#         notification_status, detailed_status = response_parser(status, detailed_status_code)
-#         current_app.logger.info(
-#             f'{client_name} callback returned status of {notification_status}'
-#             f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
-#         )
-#     except KeyError:
-#         _process_for_status(
-#             notification_status='technical-failure',
-#             client_name=client_name,
-#             provider_reference=provider_reference
-#         )
-#         raise ClientException(f'{client_name} callback failed: status {status} not found.')
+    # validate status
+    try:
+        notification_status, detailed_status = response_parser(status, detailed_status_code)
+        current_app.logger.info(
+            f'{client_name} callback returned status of {notification_status}'
+            f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
+        )
+    except KeyError:
+        _process_for_status(
+            notification_status='technical-failure',
+            client_name=client_name,
+            provider_reference=provider_reference
+        )
+        raise ClientException(f'{client_name} callback failed: status {status} not found.')
 
-#     _process_for_status(
-#         notification_status=notification_status,
-#         client_name=client_name,
-#         provider_reference=provider_reference,
-#         detailed_status_code=detailed_status_code
-#     )
+    _process_for_status(
+        notification_status=notification_status,
+        client_name=client_name,
+        provider_reference=provider_reference,
+        detailed_status_code=detailed_status_code
+    )
 
 
 def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):
@@ -68,7 +72,7 @@ def _process_for_status(notification_status, client_name, provider_reference, de
 
     if notification.sent_at:
         statsd_client.timing_with_dates(
-            f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
+            'callback.{}.elapsed-time'.format(client_name.lower()),
             datetime.utcnow(),
             notification.sent_at
         )
@@ -87,4 +91,9 @@ def _process_for_status(notification_status, client_name, provider_reference, de
         notifications_dao.dao_update_notification(notification)
 
     if notification_status != NOTIFICATION_PENDING:
-        check_and_queue_callback_task(notification)
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        # queue callback task only if the service_callback_api exists
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)

app/celery/provider_tasks.py

@@ -1,11 +1,11 @@
 from flask import current_app
+from notifications_utils.recipients import InvalidEmailError
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
-from app.clients.email import EmailClientNonRetryableException
-from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
-from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
+from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.delivery import send_to_providers
@@ -14,6 +14,7 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE
 
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
     try:
         current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -21,18 +22,11 @@ def deliver_sms(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_sms_to_provider(notification)
-    except Exception as e:
-        if isinstance(e, SmsClientResponseException):
-            current_app.logger.warning(
-                "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True
-            )
-        else:
+    except Exception:
+        try:
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
-
-        try:
             if self.request.retries == 0:
                 self.retry(queue=QueueNames.RETRY, countdown=0)
             else:
@@ -45,6 +39,7 @@ def deliver_sms(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
     try:
         current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
@@ -52,10 +47,8 @@ def deliver_email(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_email_to_provider(notification)
-    except EmailClientNonRetryableException as e:
-        current_app.logger.exception(
-            f"Email notification {notification_id} failed: {e}"
-        )
+    except InvalidEmailError as e:
+        current_app.logger.exception(e)
         update_notification_status_by_id(notification_id, 'technical-failure')
     except Exception as e:
         try:

app/celery/reporting_tasks.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
 
 from app import notify_celery
@@ -8,16 +9,21 @@ from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_billing_dao import (
     fetch_billing_data_for_day,
-    update_fact_billing,
+    update_fact_billing
+)
+from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    LETTER_TYPE,
 )
-from app.dao.fact_notification_status_dao import update_fact_notification_status
-from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE
 
 
 @notify_celery.task(name="create-nightly-billing")
 @cronitor("create-nightly-billing")
+@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
+    current_app.logger.info("create-nightly-billing task: started")
     # day_start is a datetime.date() object. e.g.
     # up to 4 days of data counting back from day_start is consolidated
     if day_start is None:
@@ -25,7 +31,7 @@ def create_nightly_billing(day_start=None):
     else:
         # When calling the task its a string in the format of "YYYY-MM-DD"
         day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 10):
+    for i in range(0, 4):
         process_day = (day_start - timedelta(days=i)).isoformat()
 
         create_nightly_billing_for_day.apply_async(
@@ -38,6 +44,7 @@ def create_nightly_billing(day_start=None):
 
 
 @notify_celery.task(name="create-nightly-billing-for-day")
+@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
     current_app.logger.info(
@@ -63,67 +70,55 @@ def create_nightly_billing_for_day(process_day):
 
 @notify_celery.task(name="create-nightly-notification-status")
 @cronitor("create-nightly-notification-status")
+@statsd(namespace="tasks")
 def create_nightly_notification_status():
-    """
-    Aggregate notification statuses into rows in ft_notification_status.
-    In order to minimise effort, this task assumes that:
-
-      - Email + SMS statuses don't change after 3 days. This is currently true
-        because all outstanding email / SMS are "timed out" after 3 days, and
-        we reject delivery receipts after this point.
-
-      - Letter statuses don't change after 9 days. There's no "timeout" for
-        letters but this is the longest we've had to cope with in the past - due
-        to major issues with our print provider.
-
-    Because the time range of the task exceeds the minimum possible retention
-    period (3 days), we need to choose which table to query for each service.
-
-    The aggregation happens for 1 extra day in case:
-
-      - This task or the "timeout" task fails to run.
-
-      - Data is (somehow) still in transit to the history table, which would
-        mean the aggregated results are temporarily incorrect.
-    """
-
+    current_app.logger.info("create-nightly-notification-status task: started")
     yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
 
-    for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
-        days = 10 if notification_type == LETTER_TYPE else 4
-
-        for i in range(days):
-            process_day = yesterday - timedelta(days=i)
-
-            relevant_service_ids = get_service_ids_with_notifications_on_date(
-                notification_type, process_day
+    # email and sms
+    for i in range(4):
+        process_day = yesterday - timedelta(days=i)
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
             )
-
-            for service_id in relevant_service_ids:
-                create_nightly_notification_status_for_service_and_day.apply_async(
-                    kwargs={
-                        'process_day': process_day.isoformat(),
-                        'notification_type': notification_type,
-                        'service_id': service_id,
-                    },
-                    queue=QueueNames.REPORTING
-                )
+            current_app.logger.info(
+                f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
+                f"for type {notification_type} for {process_day}"
+            )
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
+    for i in range(10):
+        process_day = yesterday - timedelta(days=i)
+        create_nightly_notification_status_for_day.apply_async(
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )
+        current_app.logger.info(
+            f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
+            f"for type letter for {process_day}"
+        )
 
 
-@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
-def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
+@notify_celery.task(name="create-nightly-notification-status-for-day")
+@statsd(namespace="tasks")
+def create_nightly_notification_status_for_day(process_day, notification_type):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: started'
+    )
 
     start = datetime.utcnow()
-    update_fact_notification_status(
-        process_day=process_day,
-        notification_type=notification_type,
-        service_id=service_id
-    )
-
+    transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
     end = datetime.utcnow()
     current_app.logger.info(
-        f'create-nightly-notification-status-for-service-and-day task update '
-        f'for {service_id}, {notification_type} for {process_day}: '
-        f'updated in {(end - start).seconds} seconds'
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
+        f'data fetched in {(end - start).seconds} seconds'
+    )
+
+    update_fact_notification_status(transit_data, process_day, notification_type)
+
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
+        f'task complete - {len(transit_data)} rows updated'
     )

app/celery/research_mode_tasks.py

@@ -1,16 +1,17 @@
-import json
 import random
 from datetime import datetime, timedelta
+import json
 
 from flask import current_app
+from requests import request, RequestException, HTTPError
+
 from notifications_utils.s3 import s3upload
-from requests import HTTPError, request
 
 from app import notify_celery
 from app.aws.s3 import file_exists
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
 from app.models import SMS_TYPE
+from app.config import QueueNames
+from app.celery.process_ses_receipts_tasks import process_ses_results
 
 temp_fail = "7700900003"
 perm_fail = "7700900002"
@@ -65,14 +66,16 @@ def make_request(notification_type, provider, data, headers):
             timeout=60
         )
         response.raise_for_status()
-    except HTTPError as e:
+    except RequestException as e:
+        api_error = HTTPError(e)
         current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
+            "API {} request on {} failed with {}".format(
+                "POST",
                 api_call,
-                e.response.status_code
+                api_error.response
             )
         )
-        raise e
+        raise api_error
     finally:
         current_app.logger.info("Mocked provider callback request finished")
     return response.json()
@@ -164,7 +167,7 @@ def ses_notification_callback(reference):
             'processingTimeMillis': 2003,
             'recipients': ['success@simulator.amazonses.com'],
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
             'smtpResponse': '250 2.6.0 Message received',
             'timestamp': '2017-11-17T12:14:03.646Z'
         },
@@ -201,7 +204,7 @@ def ses_notification_callback(reference):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:01.643Z'
         },
@@ -211,14 +214,14 @@ def ses_notification_callback(reference):
     return {
         'Type': 'Notification',
         'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:03.710Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
         'MessageAttributes': {}
     }
 
@@ -244,7 +247,7 @@ def _ses_bounce_callback(reference, bounce_type):
             }],
             'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
             'timestamp': '2017-11-17T12:14:05.131Z'
         },
         'mail': {
@@ -280,7 +283,7 @@ def _ses_bounce_callback(reference, bounce_type):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:03.000Z'
         },
@@ -289,13 +292,13 @@ def _ses_bounce_callback(reference, bounce_type):
     return {
         'Type': 'Notification',
         'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:05.149Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
         'MessageAttributes': {}
     }

app/celery/scheduled_tasks.py

@@ -1,66 +1,54 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from sqlalchemy import between
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import and_
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import db, notify_celery, zendesk_client
-from app.aws import s3
-from app.celery.broadcast_message_tasks import trigger_link_test
-from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app import notify_celery, zendesk_client
 from app.celery.tasks import (
-    get_recipient_csv_and_template_and_sender_id,
-    process_incomplete_jobs,
     process_job,
+    get_recipient_csv_and_template_and_sender_id,
     process_row,
-)
-from app.config import QueueNames, TaskNames
-from app.dao.invited_org_user_dao import (
-    delete_org_invitations_created_more_than_two_days_ago,
-)
-from app.dao.invited_user_dao import (
-    delete_invitations_created_more_than_two_days_ago,
-)
+    process_incomplete_jobs)
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app.config import QueueNames
+from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
     find_jobs_with_missing_rows,
-    find_missing_row_for_job,
+    find_missing_row_for_job
 )
+from app.dao.jobs_dao import dao_update_job
 from app.dao.notifications_dao import (
-    dao_old_letters_with_created_status,
-    dao_precompiled_letters_still_pending_virus_check,
-    is_delivery_slow_for_providers,
-    letters_missing_from_sending_bucket,
     notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket,
+    is_delivery_slow_for_providers,
 )
 from app.dao.provider_details_dao import (
-    dao_adjust_provider_priority_back_to_resting_points,
     dao_reduce_sms_provider_priority,
-)
-from app.dao.services_dao import (
-    dao_find_services_sending_to_tv_numbers,
-    dao_find_services_with_high_failure_rates,
+    dao_adjust_provider_priority_back_to_resting_points
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.letters.utils import generate_letter_pdf_filename
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
 from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
-    BroadcastMessage,
-    BroadcastStatusType,
     Job,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_ERROR,
+    SMS_TYPE,
+    EMAIL_TYPE,
 )
 from app.notifications.process_notifications import send_notification_to_queue
 
 
 @notify_celery.task(name="run-scheduled-jobs")
+@statsd(namespace="tasks")
 def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
@@ -72,6 +60,7 @@ def run_scheduled_jobs():
 
 
 @notify_celery.task(name="delete-verify-codes")
+@statsd(namespace="tasks")
 def delete_verify_codes():
     try:
         start = datetime.utcnow()
@@ -85,6 +74,7 @@ def delete_verify_codes():
 
 
 @notify_celery.task(name="delete-invitations")
+@statsd(namespace="tasks")
 def delete_invitations():
     try:
         start = datetime.utcnow()
@@ -99,6 +89,7 @@ def delete_invitations():
 
 
 @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
 def switch_current_sms_provider_on_slow_delivery():
     """
     Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
@@ -121,42 +112,32 @@ def switch_current_sms_provider_on_slow_delivery():
 
 
 @notify_celery.task(name='tend-providers-back-to-middle')
+@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
     dao_adjust_provider_priority_back_to_resting_points()
 
 
 @notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
 def check_job_status():
     """
     every x minutes do this check
     select
     from jobs
     where job_status == 'in progress'
-    and processing started between 30 and 35 minutes ago
-    OR where the job_status == 'pending'
-    and the job scheduled_for timestamp is between 30 and 35 minutes ago.
+    and template_type in ('sms', 'email')
+    and scheduled_at or created_at is older that 30 minutes.
     if any results then
-        update the job_status to 'error'
+        raise error
         process the rows in the csv that are missing (in another task) just do the check here.
     """
     thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
     thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
 
-    incomplete_in_progress_jobs = Job.query.filter(
+    jobs_not_complete_after_30_minutes = Job.query.filter(
         Job.job_status == JOB_STATUS_IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-    incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
-        Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-
-    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
-        incomplete_pending_jobs
-    ).order_by(
-        Job.processing_started, Job.scheduled_for
-    ).all()
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
+    ).order_by(Job.processing_started).all()
 
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
@@ -175,6 +156,7 @@ def check_job_status():
 
 
 @notify_celery.task(name='replay-created-notifications')
+@statsd(namespace="tasks")
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
     resend_created_notifications_older_than = (60 * 60)
@@ -207,72 +189,47 @@ def replay_created_notifications():
             get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)
 
 
-@notify_celery.task(name='check-if-letters-still-pending-virus-check')
-def check_if_letters_still_pending_virus_check():
-    letters = []
-
-    for letter in dao_precompiled_letters_still_pending_virus_check():
-        # find letter in the scan bucket
-        filename = generate_letter_pdf_filename(
-            letter.reference,
-            letter.created_at,
-            ignore_folder=True,
-            postage=letter.postage
-        )
-
-        if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename):
-            current_app.logger.warning(
-                f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.'
-            )
-            notify_celery.send_task(
-                name=TaskNames.SCAN_FILE,
-                kwargs={'filename': filename},
-                queue=QueueNames.ANTIVIRUS,
-            )
-        else:
-            letters.append(letter)
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()
 
     if len(letters) > 0:
         letter_ids = [(str(letter.id), letter.reference) for letter in letters]
 
-        msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes.
-            We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off
-            again or move them to technical failure.
+        msg = """{} precompiled letters have been pending-virus-check for over 90 minutes. Follow runbook to resolve:
+            https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-letter-pending-virus-scan-for-90-minutes.
+            Notifications: {}""".format(len(letters), sorted(letter_ids))
 
-            Notifications: {sorted(letter_ids)}"""
+        current_app.logger.exception(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
                 message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)
 
 
-@notify_celery.task(name='check-if-letters-still-in-created')
-def check_if_letters_still_in_created():
+@notify_celery.task(name='check-templated-letter-state')
+@statsd(namespace="tasks")
+def check_templated_letter_state():
     letters = dao_old_letters_with_created_status()
 
     if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
         msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Follow runbook to resolve: " \
-              "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
-              "#deal-with-Letters-still-in-created.".format(len(letters))
+              "Notifications: {}".format(len(letters), letter_ids)
+
+        current_app.logger.exception(msg)
 
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status",
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
                 message=msg,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True,
-                ticket_categories=['notify_letters']
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-            current_app.logger.error(msg)
 
 
 @notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
@@ -289,6 +246,7 @@ def check_for_missing_rows_in_completed_jobs():
 
 
 @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
+@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
     start_date = (datetime.utcnow() - timedelta(days=1))
     end_date = datetime.utcnow()
@@ -326,44 +284,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             message += ("\nYou can find instructions for this ticket in our manual:\n"
                         "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
+            zendesk_client.create_ticket(
+                subject="[{}] High failure rates for sms spotted for services".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']
+                ),
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)
-
-
-@notify_celery.task(name='trigger-link-tests')
-def trigger_link_tests():
-    if current_app.config['CBC_PROXY_ENABLED']:
-        for cbc_name in current_app.config['ENABLED_CBCS']:
-            trigger_link_test.apply_async(kwargs={'provider': cbc_name}, queue=QueueNames.BROADCASTS)
-
-
-@notify_celery.task(name='auto-expire-broadcast-messages')
-def auto_expire_broadcast_messages():
-    expired_broadcasts = BroadcastMessage.query.filter(
-        BroadcastMessage.finishes_at <= datetime.now(),
-        BroadcastMessage.status == BroadcastStatusType.BROADCASTING,
-    ).all()
-
-    for broadcast in expired_broadcasts:
-        broadcast.status = BroadcastStatusType.COMPLETED
-
-    db.session.commit()
-
-    if expired_broadcasts:
-        notify_celery.send_task(
-            name=TaskNames.PUBLISH_GOVUK_ALERTS,
-            queue=QueueNames.GOVUK_ALERTS
-        )
-
-
-@notify_celery.task(name='remove-yesterdays-planned-tests-on-govuk-alerts')
-def remove_yesterdays_planned_tests_on_govuk_alerts():
-    notify_celery.send_task(
-        name=TaskNames.PUBLISH_GOVUK_ALERTS,
-        queue=QueueNames.GOVUK_ALERTS
-    )

app/celery/service_callback_tasks.py

@@ -1,14 +1,22 @@
 import json
 
 from flask import current_app
-from requests import HTTPError, RequestException, request
+from notifications_utils.statsd_decorators import statsd
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
 
-from app import encryption, notify_celery
+from app import (
+    notify_celery,
+    encryption
+)
 from app.config import QueueNames
-from app.utils import DATETIME_FORMAT
 
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
@@ -22,11 +30,8 @@ def send_delivery_status_to_service(
         "created_at": status_update['notification_created_at'],
         "completed_at": status_update['notification_updated_at'],
         "sent_at": status_update['notification_sent_at'],
-        "notification_type": status_update['notification_type'],
-        "template_id": status_update['template_id'],
-        "template_version": status_update['template_version']
+        "notification_type": status_update['notification_type']
     }
-
     _send_data_to_service_callback_api(
         self,
         data,
@@ -37,6 +42,7 @@ def send_delivery_status_to_service(
 
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
     complaint = encryption.decrypt(complaint_data)
 
@@ -68,7 +74,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 'Content-Type': 'application/json',
                 'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=5
+            timeout=60
         )
         current_app.logger.info('{} sending {} to {}, response {}'.format(
             function_name,
@@ -79,16 +85,16 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            "{} request failed for notification_id: {} and url: {}. exception: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
                 function_name,
                 notification_id,
                 service_callback_url,
                 e
             )
         )
-        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
-                self.retry(queue=QueueNames.CALLBACKS_RETRY)
+                self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.warning(
                     "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
@@ -97,18 +103,10 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                         notification_id
                     )
                 )
-        else:
-            current_app.logger.warning(
-                "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name,
-                    notification_id,
-                    service_callback_url,
-                    e
-                )
-            )
 
 
 def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "notification_id": str(notification.id),
         "notification_client_reference": notification.client_reference,
@@ -121,13 +119,12 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_type": notification.notification_type,
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
-        "template_id": str(notification.template_id),
-        "template_version": notification.template_version,
     }
     return encryption.encrypt(data)
 
 
 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "complaint_id": str(complaint.id),
         "notification_id": str(notification.id),

app/celery/tasks.py

@@ -1,38 +1,49 @@
 import json
-from collections import defaultdict, namedtuple
 from datetime import datetime
+from collections import namedtuple, defaultdict
 
 from flask import current_app
-from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.columns import Columns
 from notifications_utils.postal_address import PostalAddress
 from notifications_utils.recipients import RecipientCSV
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.timezones import convert_utc_to_bst
-from requests import HTTPError, RequestException, request
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 
-from app import create_random_identifier, create_uuid, encryption, notify_celery
+from app import (
+    create_uuid,
+    create_random_identifier,
+    DATETIME_FORMAT,
+    encryption,
+    notify_celery,
+)
 from app.aws import s3
-from app.celery import letters_pdf_tasks, provider_tasks, research_mode_tasks
+from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
 from app.config import QueueNames
-from app.dao.daily_sorted_letter_dao import (
-    dao_create_or_update_daily_sorted_letter,
-)
+from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
-from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id,
+)
 from app.dao.notifications_dao import (
-    dao_get_last_notification_added_for_job_id,
-    dao_get_notification_or_history_by_reference,
-    dao_update_notifications_by_reference,
     get_notification_by_id,
+    dao_update_notifications_by_reference,
+    dao_get_last_notification_added_for_job_id,
     update_notification_status_by_reference,
+    dao_get_notification_or_history_by_reference,
 )
-from app.dao.provider_details_dao import (
-    get_provider_details_by_notification_type,
-)
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
 from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
@@ -46,22 +57,19 @@ from app.models import (
     LETTER_TYPE,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
-    NOTIFICATION_RETURNED_LETTER,
     NOTIFICATION_SENDING,
-    NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
     SMS_TYPE,
     DailySortedLetter,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
-from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, get_reference_from_personalisation
-from app.v2.errors import TooManyRequestsError
 
 
 @notify_celery.task(name="process-job")
+@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
@@ -72,10 +80,6 @@ def process_job(job_id, sender_id=None):
 
     service = job.service
 
-    job.job_status = JOB_STATUS_IN_PROGRESS
-    job.processing_started = start
-    dao_update_job(job)
-
     if not service.active:
         job.job_status = JOB_STATUS_CANCELLED
         dao_update_job(job)
@@ -86,6 +90,10 @@ def process_job(job_id, sender_id=None):
     if __sending_limits_for_job_exceeded(service, job, job_id):
         return
 
+    job.job_status = JOB_STATUS_IN_PROGRESS
+    job.processing_started = start
+    dao_update_job(job)
+
     recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
 
     current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
@@ -160,13 +168,9 @@ def process_row(row, template, job, service, sender_id=None):
 
 
 def __sending_limits_for_job_exceeded(service, job, job_id):
-    try:
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-        if total_sent + job.notification_count > service.message_limit:
-            raise TooManyRequestsError(service.message_limit)
-        else:
-            return False
-    except TooManyRequestsError:
+    total_sent = fetch_todays_total_message_count(service.id)
+
+    if total_sent + job.notification_count > service.message_limit:
         job.job_status = 'sending limits exceeded'
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
@@ -175,26 +179,24 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
                 job_id, job.notification_count, service.message_limit)
         )
         return True
+    return False
 
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_sms(self,
              service_id,
              notification_id,
              encrypted_notification,
              sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
 
     if sender_id:
         reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.debug(
@@ -236,6 +238,7 @@ def save_sms(self,
 
 
 @notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_email(self,
                service_id,
                notification_id,
@@ -243,17 +246,13 @@ def save_email(self,
                sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
 
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
 
     if sender_id:
         reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.info("Email {} failed as restricted service".format(notification_id))
@@ -287,22 +286,14 @@ def save_email(self,
 
 
 @notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
-def save_api_email(self, encrypted_notification):
+@statsd(namespace="tasks")
+def save_api_email(self,
+                   encrypted_notification,
+                   ):
 
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
-def save_api_sms(self, encrypted_notification):
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-def save_api_email_or_sms(self, encrypted_notification):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification['service_id'])
-    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
-    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
-        else provider_tasks.deliver_sms
+    service = dao_fetch_service_by_id(notification['service_id'])
+
     try:
 
         persist_notification(
@@ -312,7 +303,7 @@ def save_api_email_or_sms(self, encrypted_notification):
             recipient=notification['to'],
             service=service,
             personalisation=notification.get('personalisation'),
-            notification_type=notification['notification_type'],
+            notification_type=EMAIL_TYPE,
             client_reference=notification['client_reference'],
             api_key_id=notification.get('api_key_id'),
             key_type=KEY_TYPE_NORMAL,
@@ -322,26 +313,25 @@ def save_api_email_or_sms(self, encrypted_notification):
             document_download_count=notification['document_download_count']
         )
 
-        q = q if not service.research_mode else QueueNames.RESEARCH_MODE
-        provider_task.apply_async(
+        q = QueueNames.SEND_EMAIL if not service.research_mode else QueueNames.RESEARCH_MODE
+        provider_tasks.deliver_email.apply_async(
             [notification['id']],
             queue=q
         )
-        current_app.logger.debug(
-            f"{notification['notification_type']} {notification['id']} has been persisted and sent to delivery queue."
-        )
+        current_app.logger.debug(f"Email {notification['id']} has been persisted and sent to delivery queue.")
     except IntegrityError:
-        current_app.logger.info(f"{notification['notification_type']} {notification['id']} already exists.")
+        current_app.logger.info(f"Email {notification['id']} already exists.")
 
     except SQLAlchemyError:
 
         try:
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
-            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
+            current_app.logger.error('Max retry failed' + f"Failed to persist notification {notification['id']}")
 
 
 @notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_letter(
         self,
         service_id,
@@ -351,15 +341,11 @@ def save_letter(
     notification = encryption.decrypt(encrypted_notification)
 
     postal_address = PostalAddress.from_personalisation(
-        InsensitiveDict(notification['personalisation'])
+        Columns(notification['personalisation'])
     )
 
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
 
     try:
         # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
@@ -380,8 +366,7 @@ def save_letter(
             job_row_number=notification['row_number'],
             notification_id=notification_id,
             reference=create_random_identifier(),
-            client_reference=get_reference_from_personalisation(notification['personalisation']),
-            reply_to_text=template.reply_to_text,
+            reply_to_text=template.get_reply_to_text(),
             status=status
         )
 
@@ -404,6 +389,7 @@ def save_letter(
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
     provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
@@ -422,6 +408,7 @@ def update_letter_notifications_to_sent_to_dvla(self, notification_references):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
 def update_letter_notifications_to_error(self, notification_references):
     # This task will be called by the FTP app to update notifications as sent to DVLA
 
@@ -457,6 +444,7 @@ def handle_exception(task, notification, notification_id, exc):
 
 
 @notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
 def update_letter_notifications_statuses(self, filename):
     notification_updates = parse_dvla_file(filename)
 
@@ -473,6 +461,7 @@ def update_letter_notifications_statuses(self, filename):
 
 
 @notify_celery.task(bind=True, name="record-daily-sorted-counts")
+@statsd(namespace="tasks")
 def record_daily_sorted_counts(self, filename):
     sorted_letter_counts = defaultdict(int)
     notification_updates = parse_dvla_file(filename)
@@ -559,6 +548,7 @@ def check_billable_units(notification_update):
 
 
 @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
     inbound_api = get_service_inbound_api_for_service(service_id=service_id)
     if not inbound_api:
@@ -587,32 +577,33 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
             },
             timeout=60
         )
-        current_app.logger.debug(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
-            f"response {response.status_code}"
-        )
+        current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
+            inbound_sms_id,
+            inbound_api.url,
+            response.status_code
+        ))
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
-            f"and url: {inbound_api.url}. exception: {e}"
+            "send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
+                service_id,
+                inbound_sms_id,
+                inbound_api.url,
+                e
+            )
         )
         if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
                 self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of" +
+                    f"Retry: send_inbound_sms_to_service has retried the max number of" +
                     f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
                 )
-        else:
-            current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
-                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
-            )
 
 
 @notify_celery.task(name='process-incomplete-jobs')
+@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
     jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
 
@@ -649,6 +640,7 @@ def process_incomplete_job(job_id):
 
 
 @notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
 def process_returned_letters_list(notification_references):
     updated, updated_history = dao_update_notifications_by_reference(
         notification_references,

app/clients/__init__.py

@@ -17,7 +17,7 @@ STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'
 
 
-class NotificationProviderClients(object):
+class Clients(object):
     sms_clients = {}
     email_clients = {}
 

app/clients/cbc_proxy.py

@@ -1,299 +0,0 @@
-import json
-import uuid
-from abc import ABC, abstractmethod
-
-import boto3
-import botocore
-from flask import current_app
-from notifications_utils.template import non_gsm_characters
-from sqlalchemy.schema import Sequence
-
-from app.config import BroadcastProvider
-from app.utils import DATETIME_FORMAT, format_sequential_number
-
-# The variable names in this file have specific meaning in a CAP message
-#
-# identifier is a unique field for each CAP message
-#
-# headline is a field which we are not sure if we will use
-#
-# description is the body of the message
-
-# areas is a list of dicts, with the following items
-# * description is a string which populates the areaDesc field
-# * polygon is a list of lat/long pairs
-#
-# previous_provider_messages is a list of previous events (models.py::BroadcastProviderMessage)
-# ie a Cancel message would have a unique event but have the event of
-#    the preceeding Alert message in the previous_provider_messages field
-
-
-class CBCProxyRetryableException(Exception):
-    pass
-
-
-class CBCProxyClient:
-    _lambda_client = None
-
-    def init_app(self, app):
-        if app.config.get('CBC_PROXY_ENABLED'):
-            self._lambda_client = boto3.client(
-                'lambda',
-                region_name='us-west-2',
-                aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'],
-                aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'],
-            )
-
-    def get_proxy(self, provider):
-        proxy_classes = {
-            BroadcastProvider.EE: CBCProxyEE,
-            BroadcastProvider.THREE: CBCProxyThree,
-            BroadcastProvider.O2: CBCProxyO2,
-            BroadcastProvider.VODAFONE: CBCProxyVodafone,
-        }
-        return proxy_classes[provider](self._lambda_client)
-
-
-class CBCProxyClientBase(ABC):
-    @property
-    @abstractmethod
-    def lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def failover_lambda_name(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_ENGLISH(self):
-        pass
-
-    @property
-    @abstractmethod
-    def LANGUAGE_WELSH(self):
-        pass
-
-    def __init__(self, lambda_client):
-        self._lambda_client = lambda_client
-
-    def send_link_test(self):
-        self._send_link_test(self.lambda_name)
-        self._send_link_test(self.failover_lambda_name)
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ): pass
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        pass
-
-    # We have not implementated updating a broadcast
-    def update_and_send_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, channel, message_number=None
-    ):
-        pass
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages, headline, description, areas,
-        sent, expires, message_number=None
-    ):
-        pass
-
-    def _invoke_lambda_with_failover(self, payload):
-        result = self._invoke_lambda(self.lambda_name, payload)
-
-        if not result:
-            failover_result = self._invoke_lambda(self.failover_lambda_name, payload)
-            if not failover_result:
-                raise CBCProxyRetryableException(
-                    f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}'
-                )
-
-        return result
-
-    def _invoke_lambda(self, lambda_name, payload):
-        payload_bytes = bytes(json.dumps(payload), encoding='utf8')
-        try:
-            current_app.logger.info(
-                f"Calling lambda {lambda_name} with payload {str(payload)[:1000]}"
-            )
-
-            result = self._lambda_client.invoke(
-                FunctionName=lambda_name,
-                InvocationType='RequestResponse',
-                Payload=payload_bytes,
-            )
-        except botocore.exceptions.ClientError:
-            current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}')
-            success = False
-            return success
-
-        if result['StatusCode'] > 299:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with status code { result['StatusCode']}, {result.get('Payload')}"
-            )
-            success = False
-
-        elif 'FunctionError' in result:
-            current_app.logger.info(
-                f"Error calling lambda {lambda_name} with function error { result['Payload'].read() }"
-            )
-            success = False
-
-        else:
-            success = True
-
-        return success
-
-    def infer_language_from(self, content):
-        if non_gsm_characters(content):
-            return self.LANGUAGE_WELSH
-        return self.LANGUAGE_ENGLISH
-
-
-class CBCProxyOne2ManyClient(CBCProxyClientBase):
-    LANGUAGE_ENGLISH = 'en-GB'
-    LANGUAGE_WELSH = 'cy-GB'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_format': 'cap'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, headline, description, areas, sent, expires, channel, message_number=None
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_format': 'cap',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self,
-        identifier, previous_provider_messages,
-        sent, message_number=None
-    ):
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_format': 'cap',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-
-class CBCProxyEE(CBCProxyOne2ManyClient):
-    lambda_name = 'ee-1-proxy'
-    failover_lambda_name = 'ee-2-proxy'
-
-
-class CBCProxyThree(CBCProxyOne2ManyClient):
-    lambda_name = 'three-1-proxy'
-    failover_lambda_name = 'three-2-proxy'
-
-
-class CBCProxyO2(CBCProxyOne2ManyClient):
-    lambda_name = 'o2-1-proxy'
-    failover_lambda_name = 'o2-2-proxy'
-
-
-class CBCProxyVodafone(CBCProxyClientBase):
-    lambda_name = 'vodafone-1-proxy'
-    failover_lambda_name = 'vodafone-2-proxy'
-
-    LANGUAGE_ENGLISH = 'English'
-    LANGUAGE_WELSH = 'Welsh'
-
-    def _send_link_test(
-        self,
-        lambda_name,
-    ):
-        """
-        link test - open up a connection to a specific provider, and send them an xml payload with a <msgType> of
-        test.
-        """
-        from app import db
-        sequence = Sequence('broadcast_provider_message_number_seq')
-        sequential_number = db.session.connection().execute(sequence)
-        formatted_seq_number = format_sequential_number(sequential_number)
-
-        payload = {
-            'message_type': 'test',
-            'identifier': str(uuid.uuid4()),
-            'message_number': formatted_seq_number,
-            'message_format': 'ibag'
-        }
-
-        self._invoke_lambda(lambda_name=lambda_name, payload=payload)
-
-    def create_and_send_broadcast(
-        self, identifier, message_number, headline, description, areas, sent, expires, channel
-    ):
-        payload = {
-            'message_type': 'alert',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            'headline': headline,
-            'description': description,
-            'areas': areas,
-            'sent': sent,
-            'expires': expires,
-            'language': self.infer_language_from(description),
-            'channel': channel,
-        }
-        self._invoke_lambda_with_failover(payload=payload)
-
-    def cancel_broadcast(
-        self, identifier, previous_provider_messages, sent, message_number
-    ):
-
-        payload = {
-            'message_type': 'cancel',
-            'identifier': identifier,
-            'message_number': message_number,
-            'message_format': 'ibag',
-            "references": [
-                {
-                    "message_id": str(message.id),
-                    "message_number": format_sequential_number(message.message_number),
-                    "sent": message.created_at.strftime(DATETIME_FORMAT)
-                } for message in previous_provider_messages
-            ],
-            'sent': sent,
-        }
-        self._invoke_lambda_with_failover(payload=payload)

app/clients/document_download.py

@@ -1,4 +1,5 @@
 import requests
+
 from flask import current_app
 
 
@@ -41,9 +42,7 @@ class DocumentDownloadClient:
             # if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
             # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
             # unrecognised) should be raised as a 500 internal server error here.
-            if e.response is None:
-                raise Exception(f'Unhandled document download error: {repr(e)}')
-            elif e.response.status_code == 400:
+            if e.response.status_code == 400:
                 error = DocumentDownloadError.from_exception(e)
                 current_app.logger.info(
                     'Document download request failed with error: {}'.format(error.message)

app/clients/email/__init__.py

@@ -1,4 +1,4 @@
-from app.clients import Client, ClientException
+from app.clients import ClientException, Client
 
 
 class EmailClientException(ClientException):
@@ -8,18 +8,6 @@ class EmailClientException(ClientException):
     pass
 
 
-class EmailClientNonRetryableException(ClientException):
-    '''
-    Represents an error returned from the email client API with a 4xx response code
-    that should not be retried and should instead be marked as technical failure.
-    An example of this would be an email address that makes it through our
-    validation rules but is rejected by SES. There is no point in retrying this type as
-    it will always fail however many calls to SES. Whereas a throttling error would not
-    use this exception as it may succeed if we retry
-    '''
-    pass
-
-
 class EmailClient(Client):
     '''
     Base Email client for sending emails.
@@ -28,6 +16,5 @@ class EmailClient(Client):
     def send_email(self, *args, **kwargs):
         raise NotImplementedError('TODO Need to implement.')
 
-    @property
-    def name(self):
+    def get_name(self):
         raise NotImplementedError('TODO Need to implement.')

app/clients/email/aws_ses.py

@@ -1,15 +1,11 @@
-from time import monotonic
-
 import boto3
 import botocore
 from flask import current_app
+from time import monotonic
+from notifications_utils.recipients import InvalidEmailError
 
 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
-from app.clients.email import (
-    EmailClient,
-    EmailClientException,
-    EmailClientNonRetryableException,
-)
+from app.clients.email import (EmailClientException, EmailClient)
 
 ses_response_map = {
     'Permanent': {
@@ -59,11 +55,27 @@ class AwsSesClient(EmailClient):
     def init_app(self, region, statsd_client, *args, **kwargs):
         self._client = boto3.client('ses', region_name=region)
         super(AwsSesClient, self).__init__(*args, **kwargs)
+        self.name = 'ses'
         self.statsd_client = statsd_client
 
-    @property
-    def name(self):
-        return 'ses'
+        # events are generally undocumented, but some that might be of interest are:
+        # before-call, after-call, after-call-error, request-created, response-received
+        self._client.meta.events.register(f'request-created.ses.SendEmail', self.ses_request_created_hook)
+        self._client.meta.events.register(f'response-received.ses.SendEmail', self.ses_response_received_hook)
+
+    def ses_request_created_hook(self, **kwargs):
+        # request created may be called multiple times if the request auto-retries. We want to count all these as the
+        # same request for timing purposes, so only reset the start time if it was cleared completely
+        if self.ses_start_time == 0:
+            self.ses_start_time = monotonic()
+
+    def ses_response_received_hook(self, **kwargs):
+        # response received may be called multiple times if the request auto-retries, however, we want to count the last
+        # time it triggers for timing purposes, so always reset the elapsed time
+        self.ses_elapsed_time = monotonic() - self.ses_start_time
+
+    def get_name(self):
+        return self.name
 
     def send_email(self,
                    source,
@@ -72,6 +84,8 @@ class AwsSesClient(EmailClient):
                    body,
                    html_body='',
                    reply_to_address=None):
+        self.ses_elapsed_time = 0
+        self.ses_start_time = 0
         try:
             if isinstance(to_addresses, str):
                 to_addresses = [to_addresses]
@@ -108,7 +122,10 @@ class AwsSesClient(EmailClient):
 
             # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
             if e.response['Error']['Code'] == 'InvalidParameterValue':
-                raise EmailClientNonRetryableException(e.response['Error']['Message'])
+                raise InvalidEmailError('email: "{}" message: "{}"'.format(
+                    to_addresses[0],
+                    e.response['Error']['Message']
+                ))
             elif (
                 e.response['Error']['Code'] == 'Throttling'
                 and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
@@ -124,6 +141,8 @@ class AwsSesClient(EmailClient):
             elapsed_time = monotonic() - start_time
             current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
             self.statsd_client.timing("clients.ses.request-time", elapsed_time)
+            if self.ses_elapsed_time != 0:
+                self.statsd_client.timing("clients.ses.raw-request-time", self.ses_elapsed_time)
             self.statsd_client.incr("clients.ses.success")
             return response['MessageId']
 

app/clients/email/aws_ses_stub.py

@@ -1,10 +1,10 @@
 import json
-from time import monotonic
 
 from flask import current_app
 from requests import request
+from time import monotonic
 
-from app.clients.email import EmailClient, EmailClientException
+from app.clients.email import (EmailClientException, EmailClient)
 
 
 class AwsSesStubClientException(EmailClientException):
@@ -13,12 +13,12 @@ class AwsSesStubClientException(EmailClientException):
 
 class AwsSesStubClient(EmailClient):
     def init_app(self, region, statsd_client, stub_url):
+        self.name = 'ses'
         self.statsd_client = statsd_client
         self.url = stub_url
 
-    @property
-    def name(self):
-        return 'ses'
+    def get_name(self):
+        return self.name
 
     def send_email(self,
                    source,

app/clients/performance_platform/performance_platform_client.py

@@ -1,8 +1,9 @@
 import base64
 import json
 
-import requests
 from flask import current_app
+import requests
+
 from notifications_utils.timezones import convert_utc_to_bst
 
 

app/clients/sms/__init__.py

@@ -1,10 +1,10 @@
-from app.clients import Client, ClientException
+from app.clients import (Client, ClientException)
 
 
 class SmsClientResponseException(ClientException):
-    """
+    '''
     Base Exception for SmsClientsResponses
-    """
+    '''
 
     def __init__(self, message):
         self.message = message
@@ -14,12 +14,12 @@ class SmsClientResponseException(ClientException):
 
 
 class SmsClient(Client):
-    """
+    '''
     Base Sms client for sending smss.
-    """
+    '''
 
     def send_sms(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
 
     def get_name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

app/clients/sms/aws_sns.py

@@ -1,86 +0,0 @@
-import re
-from time import monotonic
-
-import boto3
-import botocore
-import phonenumbers
-
-from app.clients.sms import SmsClient
-
-
-class AwsSnsClient(SmsClient):
-    """
-    AwsSns sms client
-    """
-
-    def init_app(self, current_app, statsd_client, *args, **kwargs):
-        self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"])
-        self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"])
-        super(SmsClient, self).__init__(*args, **kwargs)
-        self.current_app = current_app
-        self.statsd_client = statsd_client
-        self.long_code_regex = re.compile(r"^\+1\d{10}$")
-
-    @property
-    def name(self):
-        return 'sns'
-
-    def send_sms(self, to, content, reference, sender=None, international=False):
-        matched = False
-
-        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
-            matched = True
-            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
-
-            client = self._client
-            # See documentation
-            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
-            attributes = {
-                "AWS.SNS.SMS.SMSType": {
-                    "DataType": "String",
-                    "StringValue": "Transactional",
-                }
-            }
-
-            # If sending with a long code number, we need to use another AWS region
-            # and specify the phone number we want to use as the origination number
-            send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender)
-            if send_with_dedicated_phone_number:
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": sender,
-                }
-
-            # If the number is US based, we must use a US Toll Free number to send the message
-            country = phonenumbers.region_code_for_number(match.number)
-            if country == "US":
-                client = self._long_codes_client
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
-                }
-
-            try:
-                start_time = monotonic()
-                response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
-            except botocore.exceptions.ClientError as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            except Exception as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            finally:
-                elapsed_time = monotonic() - start_time
-                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
-                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
-                self.statsd_client.incr("clients.sns.success")
-            return response["MessageId"]
-
-        if not matched:
-            self.statsd_client.incr("clients.sns.error")
-            self.current_app.logger.error("No valid numbers found in {}".format(to))
-            raise ValueError("No valid numbers found for SMS delivery")
-
-    def _send_with_dedicated_phone_number(self, sender):
-        return sender and re.match(self.long_code_regex, sender)

app/clients/sms/firetext.py

@@ -0,0 +1,130 @@
+import json
+import logging
+
+from time import monotonic
+from requests import request, RequestException
+
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+logger = logging.getLogger(__name__)
+
+# Firetext will send a delivery receipt with three different status codes.
+# The `firetext_response` maps these codes to the notification statistics status and notification status.
+# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
+# the notification status to temporary-failure rather than permanent failure.
+#  See the code in the notification_dao.update_notifications_status_by_id
+firetext_responses = {
+    '0': 'delivered',
+    '1': 'permanent-failure',
+    '2': 'pending'
+}
+
+firetext_codes = {
+    # code '000' means 'No errors reported'
+    '101': {'status': 'permanent-failure', 'reason': 'Unknown Subscriber'},
+    '102': {'status': 'temporary-failure', 'reason': 'Absent Subscriber'},
+    '103': {'status': 'temporary-failure', 'reason': 'Subscriber Busy'},
+    '104': {'status': 'temporary-failure', 'reason': 'No Subscriber Memory'},
+    '201': {'status': 'permanent-failure', 'reason': 'Invalid Number'},
+    '301': {'status': 'permanent-failure', 'reason': 'SMS Not Supported'},
+    '302': {'status': 'temporary-failure', 'reason': 'SMS Not Supported'},
+    '401': {'status': 'permanent-failure', 'reason': 'Message Rejected'},
+    '900': {'status': 'temporary-failure', 'reason': 'Routing Error'},
+}
+
+
+def get_firetext_responses(status, detailed_status_code=None):
+    detailed_status = firetext_codes[detailed_status_code]['reason'] if firetext_codes.get(
+        detailed_status_code, None
+    ) else None
+    return (firetext_responses[status], detailed_status)
+
+
+def get_message_status_and_reason_from_firetext_code(detailed_status_code):
+    return firetext_codes[detailed_status_code]['status'], firetext_codes[detailed_status_code]['reason']
+
+
+class FiretextClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class FiretextClient(SmsClient):
+    '''
+    FireText sms client.
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('FIRETEXT_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'firetext'
+        self.url = current_app.config.get('FIRETEXT_URL')
+        self.statsd_client = statsd_client
+
+    def get_name(self):
+        return self.name
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.firetext.success")
+        else:
+            self.statsd_client.incr("clients.firetext.error")
+            self.current_app.logger.error(log_message)
+
+    def send_sms(self, to, content, reference, sender=None):
+
+        data = {
+            "apiKey": self.api_key,
+            "from": self.from_number if sender is None else sender,
+            "to": to.replace('+', ''),
+            "message": content,
+            "reference": reference
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.url,
+                data=data,
+                timeout=60
+            )
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+                if response.json()['code'] != 0:
+                    raise ValueError()
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise FiretextClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise FiretextClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
+            self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.firetext.raw-request-time", response.elapsed.total_seconds())
+        return response

app/clients/sms/mmg.py

@@ -0,0 +1,140 @@
+import json
+from time import monotonic
+from requests import (request, RequestException)
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+mmg_response_map = {
+    '2': {'status': 'permanent-failure', 'substatus': {
+        "1": "Number does not exist",
+        "4": "Rejected by operator",
+        "5": "Unidentified Subscriber",
+        "9": "Undelivered",
+        "11": "Service for Subscriber suspended",
+        "12": "Illegal equipment",
+        "2049": "Subscriber IMSI blacklisted",
+        "2050": "Number blacklisted in do-not-disturb blacklist",
+        "2052": "Destination number blacklisted",
+        "2053": "Source address blacklisted"
+    }},
+    '3': {'status': 'delivered', 'substatus': {"2": "Delivered to operator", "5": "Delivered to handset"}},
+    '4': {'status': 'temporary-failure', 'substatus': {
+        "6": "Absent Subscriber",
+        "8": "Roaming not allowed",
+        "13": "SMS Not Supported",
+        "15": "Expired",
+        "27": "Absent Subscriber",
+        "29": "Invalid delivery report",
+        "32": "Delivery Failure",
+    }},
+    '5': {'status': 'permanent-failure', 'substatus': {
+        "6": "Network out of coverage",
+        "8": "Incorrect number prefix",
+        "10": "Number on do-not-disturb service",
+        "11": "Sender id not registered",
+        "13": "Sender id blacklisted",
+        "14": "Destination number blacklisted",
+        "19": "Routing unavailable",
+        "20": "Rejected by anti-flooding mechanism",
+        "21": "System error",  # it says to retry those messages or contact support
+        "23": "Duplicate message id",
+        "24": "Message formatted incorrectly",
+        "25": "Message too long",
+        "51": "Missing recipient value",
+        "52": "Invalid destination",
+    }},
+}
+
+
+def get_mmg_responses(status, detailed_status_code=None):
+    return (mmg_response_map[status]["status"], mmg_response_map[status]["substatus"].get(detailed_status_code, None))
+
+
+class MMGClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class MMGClient(SmsClient):
+    '''
+    MMG sms client
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('MMG_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'mmg'
+        self.statsd_client = statsd_client
+        self.mmg_url = current_app.config.get('MMG_URL')
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.mmg_url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.mmg.success")
+        else:
+            self.statsd_client.incr("clients.mmg.error")
+            self.current_app.logger.error(log_message)
+
+    def get_name(self):
+        return self.name
+
+    def send_sms(self, to, content, reference, multi=True, sender=None):
+        data = {
+            "reqType": "BULK",
+            "MSISDN": to,
+            "msg": content,
+            "sender": self.from_number if sender is None else sender,
+            "cid": reference,
+            "multi": multi
+        }
+
+        response = None
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.mmg_url,
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Basic {}'.format(self.api_key)
+                },
+                timeout=60
+            )
+
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise MMGClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise MMGClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
+            if response and hasattr(response, 'elapsed'):
+                self.statsd_client.timing("clients.mmg.raw-request-time", response.elapsed.total_seconds())
+
+            self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
+
+        return response

app/cloudfoundry_config.py

@@ -1,12 +1,21 @@
-import json
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
 import os
+import json
 
 
 def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+    set_config_env_vars(vcap_services)
 
+
+def set_config_env_vars(vcap_services):
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'].replace('postgres',
-                                                                                                       'postgresql')
-    # Redis config
-    os.environ['REDIS_URL'] = vcap_services['redis'][0]['credentials']['uri']
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']
+
+    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'

app/commands.py

@@ -1,33 +1,29 @@
 import csv
 import functools
-import itertools
-import os
 import uuid
 from datetime import datetime, timedelta
+from decimal import Decimal
 
 import click
 import flask
+import itertools
 from click_datetime import Datetime as click_dt
 from flask import current_app, json
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
-from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from notifications_utils.statsd_decorators import statsd
 
-from app import db
+from app import db, DATETIME_FORMAT, encryption
 from app.aws import s3
-from app.celery.letters_pdf_tasks import (
-    get_pdf_for_templated_letter,
-    resanitise_pdf,
-)
-from app.celery.tasks import process_row, record_daily_sorted_counts
+from app.celery.tasks import record_daily_sorted_counts, process_row
+from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
+from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter
+from app.celery.reporting_tasks import create_nightly_notification_status_for_day
 from app.config import QueueNames
-from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
-    set_default_free_allowance_for_service,
-)
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
 from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
@@ -35,40 +31,36 @@ from app.dao.fact_billing_dao import (
     update_fact_billing,
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.dao.organisation_dao import (
-    dao_add_service_to_organisation,
-    dao_get_organisation_by_email_address,
-    dao_get_organisation_by_id,
-)
-from app.dao.permissions_dao import permission_dao
+from app.dao.organisation_dao import dao_get_organisation_by_email_address, dao_add_service_to_organisation
+
+from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.services_dao import (
+    delete_service_and_all_associated_db_objects,
     dao_fetch_all_services_by_user,
     dao_fetch_all_services_created_by_user,
     dao_fetch_service_by_id,
-    dao_update_service,
-    delete_service_and_all_associated_db_objects,
+    dao_update_service
 )
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.users_dao import (
-    delete_model_user,
-    delete_user_verify_codes,
-    get_user_by_email,
-)
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes, get_user_by_email
 from app.models import (
-    KEY_TYPE_TEST,
+    PROVIDERS,
     NOTIFICATION_CREATED,
+    KEY_TYPE_TEST,
     SMS_TYPE,
-    AnnualBilling,
-    Domain,
-    EmailBranding,
-    LetterBranding,
+    EMAIL_TYPE,
+    LETTER_TYPE,
+    User,
     Notification,
     Organisation,
-    Permission,
+    Domain,
     Service,
-    User,
+    EmailBranding,
+    LetterBranding,
 )
-from app.utils import get_london_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before
 
 
 @click.group(name='command', help='Additional commands')
@@ -81,28 +73,32 @@ class notify_command:
         self.name = name
 
     def __call__(self, func):
-        decorators = [
-            functools.wraps(func),  # carry through function name, docstrings, etc.
-            click.command(name=self.name),  # turn it into a click.Command
-        ]
-
-        # in the test environment the app context is already provided and having
-        # another will lead to the test db connection being closed prematurely
-        if os.getenv('NOTIFY_ENVIRONMENT', '') != 'test':
-            # with_appcontext ensures the config is loaded, db connected, etc.
-            decorators.insert(0, flask.cli.with_appcontext)
-
+        # we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
+        # we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
+        # Then we need to turn it into a click.Command - that's what command_group.add_command expects.
+        @click.command(name=self.name)
+        @functools.wraps(func)
+        @flask.cli.with_appcontext
         def wrapper(*args, **kwargs):
             return func(*args, **kwargs)
 
-        for decorator in decorators:
-            # this syntax is equivalent to e.g. "@flask.cli.with_appcontext"
-            wrapper = decorator(wrapper)
-
         command_group.add_command(wrapper)
+
         return wrapper
 
 
+@notify_command()
+@click.option('-p', '--provider_name', required=True, type=click.Choice(PROVIDERS))
+@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals', type=float)
+@click.option('-d', '--valid_from', required=True, type=click_dt(format='%Y-%m-%dT%H:%M:%S'))
+def create_provider_rates(provider_name, cost, valid_from):
+    """
+    Backfill rates for a given provider
+    """
+    cost = Decimal(cost)
+    dao_create_provider_rates(provider_name, valid_from, cost)
+
+
 @notify_command()
 @click.option('-u', '--user_email_prefix', required=True, help="""
     Functional test user email prefix. eg "notify-test-preview"
@@ -223,43 +219,255 @@ def fix_notification_statuses_not_in_sync():
         result = db.session.execute(subq_hist).fetchall()
 
 
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+
+    WARNING: This does not overwrite existing data. You need to delete
+             the existing data or Performance Platform will double-count.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_processing_time(start_date, end_date):
+    """
+    Send historical processing time to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+        # because the tz conversion funcs talk about midnight, and the midnight before last,
+        # we want to pretend we're running this from the next morning, so add one.
+        process_date = start_date + timedelta(days=i + 1)
+
+        process_start_date = get_midnight_for_day_before(process_date)
+        process_end_date = get_london_midnight_in_utc(process_date)
+
+        print('Sending notification processing-time for {} - {}'.format(
+            process_start_date.isoformat(),
+            process_end_date.isoformat()
+        ))
+        send_processing_time_for_start_and_end(process_start_date, process_end_date)
+
+
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='list-routes')
+def list_routes():
+    """List URLs of all application routes."""
+    for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
+        print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
+
+
 @notify_command(name='insert-inbound-numbers')
 @click.option('-f', '--file_name', required=True,
               help="""Full path of the file to upload, file is a contains inbound numbers,
               one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+    file = open(file_name)
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
 
-        for line in file:
-            line = line.strip()
-            if line:
-                print(line)
-                db.session.execute(sql.format(uuid.uuid4(), line))
-                db.session.commit()
+    for line in file:
+        print(line)
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
+        db.session.commit()
+    file.close()
 
 
-@notify_command(name='replay-create-pdf-for-templated-letter')
+@notify_command(name='replay-create-pdf-letters')
 @click.option('-n', '--notification_id', type=click.UUID, required=True,
               help="Notification id of the letter that needs the get_pdf_for_templated_letter task replayed")
-def replay_create_pdf_for_templated_letter(notification_id):
+def replay_create_pdf_letters(notification_id):
     print("Create task to get_pdf_for_templated_letter for notification: {}".format(notification_id))
     get_pdf_for_templated_letter.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)
 
 
-@notify_command(name='recreate-pdf-for-precompiled-or-uploaded-letter')
-@click.option('-n', '--notification_id', type=click.UUID, required=True,
-              help="Notification ID of the precompiled or uploaded letter")
-def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id):
-    print(f"Call resanitise_pdf task for notification: {notification_id}")
-    resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS)
+@notify_command(name='replay-service-callbacks')
+@click.option('-f', '--file_name', required=True,
+              help="""Full path of the file to upload, file is a contains client references of
+              notifications that need the status to be sent to the service.""")
+@click.option('-s', '--service_id', required=True,
+              help="""The service that the callbacks are for""")
+def replay_service_callbacks(file_name, service_id):
+    print("Start send service callbacks for service: ", service_id)
+    callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
+    if not callback_api:
+        print("Callback api was not found for service: {}".format(service_id))
+        return
+
+    errors = []
+    notifications = []
+    file = open(file_name)
+
+    for ref in file:
+        try:
+            notification = Notification.query.filter_by(client_reference=ref.strip()).one()
+            notifications.append(notification)
+        except NoResultFound:
+            errors.append("Reference: {} was not found in notifications.".format(ref))
+
+    for e in errors:
+        print(e)
+    if errors:
+        raise Exception("Some notifications for the given references were not found")
+
+    for n in notifications:
+        data = {
+            "notification_id": str(n.id),
+            "notification_client_reference": n.client_reference,
+            "notification_to": n.to,
+            "notification_status": n.status,
+            "notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
+            "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
+            "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
+            "notification_type": n.notification_type,
+            "service_callback_api_url": callback_api.url,
+            "service_callback_api_bearer_token": callback_api.bearer_token,
+        }
+        encrypted_status_update = encryption.encrypt(data)
+        send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
+                                                    queue=QueueNames.CALLBACKS)
+
+    print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
+        service_id, len(notifications)))
 
 
 def setup_commands(application):
     application.cli.add_command(command_group)
 
 
+@notify_command(name='migrate-data-to-ft-billing')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@statsd(namespace="tasks")
+def migrate_data_to_ft_billing(start_date, end_date):
+
+    current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
+
+    process_date = start_date
+    total_updated = 0
+
+    while process_date < end_date:
+        start_time = datetime.utcnow()
+        # migrate data into ft_billing, upserting the data if it the record already exists
+        sql = \
+            """
+            insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
+                international, billable_units, notifications_sent, rate, postage, created_at)
+                select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
+                    case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
+                from (
+                    select
+                        n.id,
+                        (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
+                        coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
+                        coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
+                        n.notification_type,
+                        coalesce(n.sent_by, (
+                        case
+                        when notification_type = 'sms' then
+                            coalesce(sent_by, 'unknown')
+                        when notification_type = 'letter' then
+                            coalesce(sent_by, 'dvla')
+                        else
+                            coalesce(sent_by, 'ses')
+                        end )) as provider,
+                        coalesce(n.rate_multiplier,1) as rate_multiplier,
+                        s.crown,
+                        coalesce((select rates.rate from rates
+                        where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
+                        order by rates.valid_from desc limit 1), 0) as sms_rate,
+                        coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
+                        and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
+                        and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
+                        as letter_rate,
+                        coalesce(n.international, false) as international,
+                        n.billable_units,
+                        1 as notifications_sent,
+                        coalesce(n.postage, 'none') as postage,
+                        now() as created_at
+                    from public.notification_history n
+                    left join services s on s.id = n.service_id
+                    where n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
+                        and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
+                        at time zone 'UTC'
+                        and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
+                    ) as individual_record
+                group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sms_rate, letter_rate, postage, created_at
+                order by bst_date
+            on conflict on constraint ft_billing_pkey do update set
+             billable_units = excluded.billable_units,
+             notifications_sent = excluded.notifications_sent,
+             rate = excluded.rate,
+             updated_at = now()
+            """
+
+        result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
+        db.session.commit()
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
+
+        process_date += timedelta(days=1)
+
+        total_updated += result.rowcount
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
 @notify_command(name='rebuild-ft-billing-for-day')
 @click.option('-s', '--service_id', required=False, type=click.UUID)
 @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -299,6 +507,29 @@ def rebuild_ft_billing_for_day(service_id, day):
             rebuild_ft_data(day, row.service_id)
 
 
+@notify_command(name='migrate-data-to-ft-notification-status')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
+@statsd(namespace="tasks")
+def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
+    notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
+
+    start_date = start_date.date()
+    end_date = end_date.date()
+    for day_diff in range((end_date - start_date).days + 1):
+        process_day = start_date + timedelta(days=day_diff)
+        for notification_type in notification_types:
+            print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
+                process_day,
+                notification_type
+            ))
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
+            )
+
+
 @notify_command(name='bulk-invite-user-to-service')
 @click.option('-f', '--file_name', required=True,
               help="Full path of the file containing a list of email address for people to invite to a service")
@@ -315,7 +546,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi
     #  platform_admin
     #  view_activity
     # "send_texts,send_emails,send_letters,view_activity"
-    from app.service_invite.rest import create_invited_user
+    from app.invite.rest import create_invited_user
     file = open(file_name)
     for email_address in file:
         data = {
@@ -522,39 +753,6 @@ def populate_organisations_from_file(file_name):
                         db.session.rollback()
 
 
-@notify_command(name='populate-organisation-agreement-details-from-file')
-@click.option('-f', '--file_name', required=True,
-              help="CSV file containing id, agreement_signed_version, "
-              "agreement_signed_on_behalf_of_name, agreement_signed_at")
-def populate_organisation_agreement_details_from_file(file_name):
-    """
-    The input file should be a comma separated CSV file with a header row and 4 columns
-    id: the organisation ID
-    agreement_signed_version
-    agreement_signed_on_behalf_of_name
-    agreement_signed_at: The date the agreement was signed in the format of 'dd/mm/yyyy'
-    """
-    with open(file_name) as f:
-        csv_reader = csv.reader(f)
-
-        # ignore the header row
-        next(csv_reader)
-
-        for row in csv_reader:
-            org = dao_get_organisation_by_id(row[0])
-
-            current_app.logger.info(f"Updating {org.name}")
-
-            assert org.agreement_signed
-
-            org.agreement_signed_version = float(row[1])
-            org.agreement_signed_on_behalf_of_name = row[2].strip()
-            org.agreement_signed_at = datetime.strptime(row[3], "%d/%m/%Y")
-
-            db.session.add(org)
-            db.session.commit()
-
-
 @notify_command(name='get-letter-details-from-zips-sent-file')
 @click.argument('file_paths', required=True, nargs=-1)
 @statsd(namespace="tasks")
@@ -729,115 +927,3 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
-
-
-@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing_with_the_previous_years_allowance(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
-@notify_command(name='populate-annual-billing-with-defaults')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2021""")
-@click.option('-m', '--missing-services-only', default=True, type=bool,
-              help="""If true then only populate services missing from annual billing for the year.
-                      If false populate the default values for all active services.""")
-def populate_annual_billing_with_defaults(year, missing_services_only):
-    """
-    Add or update annual billing with free allowance defaults for all active services.
-    The default free allowance limits are in: app/dao/annual_billing_dao.py:57.
-
-    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
-    This is useful to prevent overriding any services that have a free allowance that is not the default.
-
-    If missing_services_only is false then add or update annual billing for all active services.
-    This is useful to ensure all services start the new year with the correct annual billing.
-    """
-    if missing_services_only:
-        active_services = Service.query.filter(
-            Service.active
-        ).outerjoin(
-            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
-        ).filter(
-            AnnualBilling.id == None  # noqa
-        ).all()
-    else:
-        active_services = Service.query.filter(
-            Service.active
-        ).all()
-    previous_year = year - 1
-    services_with_zero_free_allowance = db.session.query(AnnualBilling.service_id).filter(
-        AnnualBilling.financial_year_start == previous_year,
-        AnnualBilling.free_sms_fragment_limit == 0
-    ).all()
-
-    for service in active_services:
-
-        # If a service has free_sms_fragment_limit for the previous year
-        # set the free allowance for this year to 0 as well.
-        # Else use the default free allowance for the service.
-        if service.id in [x.service_id for x in services_with_zero_free_allowance]:
-            print(f'update service {service.id} to 0')
-            dao_create_or_update_annual_billing_for_year(
-                service_id=service.id,
-                free_sms_fragment_limit=0,
-                financial_year_start=year
-            )
-        else:
-            print(f'update service {service.id} with default')
-            set_default_free_allowance_for_service(service, year)
-
-
-@click.option('-u', '--user-id', required=True)
-@notify_command(name='local-dev-broadcast-permissions')
-def local_dev_broadcast_permissions(user_id):
-    if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
-    user = User.query.filter_by(id=user_id).one()
-
-    user_broadcast_services = Service.query.filter(
-        Service.permissions.any(permission='broadcast'),
-        Service.users.any(id=user_id)
-    )
-
-    for service in user_broadcast_services:
-        permission_list = [
-            Permission(service_id=service.id, user_id=user_id, permission=permission)
-            for permission in [
-                'reject_broadcasts', 'cancel_broadcasts',  # required to create / approve
-                'create_broadcasts', 'approve_broadcasts',  # minimum for testing
-                'manage_templates',  # unlikely but might be useful
-                'view_activity',  # normally added on invite / service creation
-            ]
-        ]
-
-        permission_dao.set_user_service_permission(
-            user, service, permission_list, _commit=True, replace=True
-        )

app/complaint/complaint_rest.py

@@ -3,10 +3,7 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request
 
 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import (
-    fetch_count_of_complaints,
-    fetch_paginated_complaints,
-)
+from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
 from app.errors import register_errors
 from app.schema_validation import validate
 from app.utils import pagination_links

app/complaint/complaint_schema.py

@@ -1,6 +1,6 @@
 
 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "complaint count request schema",
     "type": "object",
     "title": "Complaint count request",

app/config.py

@@ -1,6 +1,6 @@
-import json
-import os
 from datetime import timedelta
+import os
+import json
 
 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -27,15 +27,11 @@ class QueueNames(object):
     PROCESS_FTP = 'process-ftp-tasks'
     CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
     CALLBACKS = 'service-callbacks'
-    CALLBACKS_RETRY = 'service-callbacks-retry'
     LETTERS = 'letter-tasks'
     SMS_CALLBACKS = 'sms-callbacks'
     ANTIVIRUS = 'antivirus-tasks'
     SANITISE_LETTERS = 'sanitise-letter-tasks'
     SAVE_API_EMAIL = 'save-api-email-tasks'
-    SAVE_API_SMS = 'save-api-sms-tasks'
-    BROADCASTS = 'broadcast-tasks'
-    GOVUK_ALERTS = 'govuk-alerts'
 
     @staticmethod
     def all_queues():
@@ -52,32 +48,18 @@ class QueueNames(object):
             QueueNames.NOTIFY,
             QueueNames.CREATE_LETTERS_PDF,
             QueueNames.CALLBACKS,
-            QueueNames.CALLBACKS_RETRY,
             QueueNames.LETTERS,
             QueueNames.SMS_CALLBACKS,
-            QueueNames.SAVE_API_EMAIL,
-            QueueNames.SAVE_API_SMS,
-            QueueNames.BROADCASTS,
+            QueueNames.SAVE_API_EMAIL
         ]
 
 
-class BroadcastProvider:
-    EE = 'ee'
-    VODAFONE = 'vodafone'
-    THREE = 'three'
-    O2 = 'o2'
-
-    PROVIDERS = [EE, VODAFONE, THREE, O2]
-
-
 class TaskNames(object):
     PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
     ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
     SCAN_FILE = 'scan-file'
     SANITISE_LETTER = 'sanitise-and-upload-letter'
     CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter'
-    PUBLISH_GOVUK_ALERTS = 'publish-govuk-alerts'
-    RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter'
 
 
 class Config(object):
@@ -88,41 +70,34 @@ class Config(object):
     API_HOST_NAME = os.getenv('API_HOST_NAME')
 
     # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = 'notify-admin'
-    GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts'
-
-    INTERNAL_CLIENT_API_KEYS = json.loads(
-        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{}')
-    )
+    API_INTERNAL_SECRETS = json.loads(os.environ.get('API_INTERNAL_SECRETS', '[]'))
 
     # encyption secret/salt
-    ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET', 'dev-notify-secret-key')
-    SECRET_KEY = os.getenv('SECRET_KEY', 'dev-notify-secret-key')
-    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT', 'dev-notify-salt ')
+    SECRET_KEY = os.getenv('SECRET_KEY')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
 
     # DB conection string
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
-    
-    # AWS SMS
-    AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2")
-    AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782")
 
     # MMG API Key
     MMG_API_KEY = os.getenv('MMG_API_KEY')
 
     # Firetext API Key
     FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
-    FIRETEXT_INTERNATIONAL_API_KEY = os.getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
 
     # Prefix to identify queues in SQS
     NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
 
     # URL of redis instance
     REDIS_URL = os.getenv('REDIS_URL')
-    REDIS_ENABLED = True
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = False
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
+
     # Zendesk
     ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')
 
@@ -137,12 +112,16 @@ class Config(object):
     # Antivirus
     ANTIVIRUS_ENABLED = True
 
+    # Broadcast Messaging
+    CBC_PROXY_URL = None
+
     ###########################
     # Default config values ###
     ###########################
 
     NOTIFY_ENVIRONMENT = 'development'
-    AWS_REGION = 'us-west-2'
+    ADMIN_CLIENT_USER_NAME = 'notify-admin'
+    AWS_REGION = 'eu-west-1'
     INVITATION_EXPIRATION_DAYS = 2
     NOTIFY_APP_NAME = 'api'
     SQLALCHEMY_RECORD_QUERIES = False
@@ -155,10 +134,7 @@ class Config(object):
     API_PAGE_SIZE = 250
     TEST_MESSAGE_FILENAME = 'Test message'
     ONE_OFF_MESSAGE_FILENAME = 'Report'
-    MAX_VERIFY_CODE_COUNT = 5
-    MAX_FAILED_LOGIN_COUNT = 10
-    
-    SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service
+    MAX_VERIFY_CODE_COUNT = 10
 
     # be careful increasing this size without being sure that we won't see slowness in pysftp
     MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
@@ -168,8 +144,8 @@ class Config(object):
 
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {
-        'mmg': 50,
-        'firetext': 50
+        'mmg': 60,
+        'firetext': 40
     }
 
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
@@ -190,167 +166,148 @@ class Config(object):
     MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
     MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
-    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
-    # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
+    MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'
 
-    CELERY = {
-        'broker_url': 'sqs://',
-        'broker_transport_options': {
-            'region': AWS_REGION,
-            'visibility_timeout': 310,
-            'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
-        },
-        'timezone': 'Europe/London',
-        'imports': [
-            'app.celery.tasks',
-            'app.celery.scheduled_tasks',
-            'app.celery.reporting_tasks',
-            'app.celery.nightly_tasks',
-        ],
-        # this is overriden by the -Q command, but locally, we should read from all queues
-        'task_queues': [
-            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
-        ],
-        'beat_schedule': {
-            # app/celery/scheduled_tasks.py
-            'run-scheduled-jobs': {
-                'task': 'run-scheduled-jobs',
-                'schedule': crontab(minute='0,15,30,45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-verify-codes': {
-                'task': 'delete-verify-codes',
-                'schedule': timedelta(minutes=63),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-invitations': {
-                'task': 'delete-invitations',
-                'schedule': timedelta(minutes=66),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'switch-current-sms-provider-on-slow-delivery': {
-                'task': 'switch-current-sms-provider-on-slow-delivery',
-                'schedule': crontab(),  # Every minute
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-job-status': {
-                'task': 'check-job-status',
-                'schedule': crontab(),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'tend-providers-back-to-middle': {
-                'task': 'tend-providers-back-to-middle',
-                'schedule': crontab(minute='*/5'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-missing-rows-in-completed-jobs': {
-                'task': 'check-for-missing-rows-in-completed-jobs',
-                'schedule': crontab(minute='*/10'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'replay-created-notifications': {
-                'task': 'replay-created-notifications',
-                'schedule': crontab(minute='0, 15, 30, 45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # app/celery/nightly_tasks.py
-            'timeout-sending-notifications': {
-                'task': 'timeout-sending-notifications',
-                'schedule': crontab(hour=0, minute=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'create-nightly-billing': {
-                'task': 'create-nightly-billing',
-                'schedule': crontab(hour=0, minute=15),
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'create-nightly-notification-status': {
-                'task': 'create-nightly-notification-status',
-                'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-notifications-older-than-retention': {
-                'task': 'delete-notifications-older-than-retention',
-                'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-inbound-sms': {
-                'task': 'delete-inbound-sms',
-                'schedule': crontab(hour=1, minute=40),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'save-daily-notification-processing-time': {
-                'task': 'save-daily-notification-processing-time',
-                'schedule': crontab(hour=2, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove_sms_email_jobs': {
-                'task': 'remove_sms_email_jobs',
-                'schedule': crontab(hour=4, minute=0),
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'remove_letter_jobs': {
-                'task': 'remove_letter_jobs',
-                'schedule': crontab(hour=4, minute=20),
-                # since we mark jobs as archived
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'check-if-letters-still-in-created': {
-                'task': 'check-if-letters-still-in-created',
-                'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-if-letters-still-pending-virus-check': {
-                'task': 'check-if-letters-still-pending-virus-check',
-                'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
-                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
-                'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-letter-notifications-still-sending': {
-                'task': 'raise-alert-if-letter-notifications-still-sending',
-                'schedule': crontab(hour=17, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # The collate-letter-pdf does assume it is called in an hour that BST does not make a
-            # difference to the truncate date which translates to the filename to process
-            'collate-letter-pdfs-to-be-sent': {
-                'task': 'collate-letter-pdfs-to-be-sent',
-                'schedule': crontab(hour=17, minute=50),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'raise-alert-if-no-letter-ack-file': {
-                'task': 'raise-alert-if-no-letter-ack-file',
-                'schedule': crontab(hour=23, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'trigger-link-tests': {
-                'task': 'trigger-link-tests',
-                'schedule': timedelta(minutes=15),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'auto-expire-broadcast-messages': {
-                'task': 'auto-expire-broadcast-messages',
-                'schedule': timedelta(minutes=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove-yesterdays-planned-tests-on-govuk-alerts': {
-                'task': 'remove-yesterdays-planned-tests-on-govuk-alerts',
-                'schedule': crontab(hour=00, minute=00),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-        }
+    BROKER_URL = 'sqs://'
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 310,
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
     }
-
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = 'Europe/London'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TASK_SERIALIZER = 'json'
+    # on reporting worker, restart workers after each task is executed to help prevent memory leaks
+    CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD')
     # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
     if os.getenv('CELERYD_PREFETCH_MULTIPLIER'):
-        CELERY['worker_prefetch_multiplier'] = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
+        CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
+    CELERY_IMPORTS = (
+        'app.celery.tasks',
+        'app.celery.scheduled_tasks',
+        'app.celery.reporting_tasks',
+        'app.celery.nightly_tasks',
+    )
+    CELERYBEAT_SCHEDULE = {
+        # app/celery/scheduled_tasks.py
+        'run-scheduled-jobs': {
+            'task': 'run-scheduled-jobs',
+            'schedule': crontab(minute='0,15,30,45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-verify-codes': {
+            'task': 'delete-verify-codes',
+            'schedule': timedelta(minutes=63),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-invitations': {
+            'task': 'delete-invitations',
+            'schedule': timedelta(minutes=66),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-job-status': {
+            'task': 'check-job-status',
+            'schedule': crontab(),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'tend-providers-back-to-middle': {
+            'task': 'tend-providers-back-to-middle',
+            'schedule': crontab(minute='*/5'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-missing-rows-in-completed-jobs': {
+            'task': 'check-for-missing-rows-in-completed-jobs',
+            'schedule': crontab(minute='*/10'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'replay-created-notifications': {
+            'task': 'replay-created-notifications',
+            'schedule': crontab(minute='0, 15, 30, 45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # app/celery/nightly_tasks.py
+        'timeout-sending-notifications': {
+            'task': 'timeout-sending-notifications',
+            'schedule': crontab(hour=0, minute=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=0, minute=15),
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'create-nightly-notification-status': {
+            'task': 'create-nightly-notification-status',
+            'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'delete-notifications-older-than-retention': {
+            'task': 'delete-notifications-older-than-retention',
+            'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(hour=1, minute=40),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'remove_sms_email_jobs': {
+            'task': 'remove_sms_email_jobs',
+            'schedule': crontab(hour=4, minute=0),
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'remove_letter_jobs': {
+            'task': 'remove_letter_jobs',
+            'schedule': crontab(hour=4, minute=20),
+            # since we mark jobs as archived
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+            'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+            'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-letter-notifications-still-sending': {
+            'task': 'raise-alert-if-letter-notifications-still-sending',
+            'schedule': crontab(hour=15, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-to-be-sent': {
+            'task': 'collate-letter-pdfs-to-be-sent',
+            'schedule': crontab(hour=17, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-no-letter-ack-file': {
+            'task': 'raise-alert-if-no-letter-ack-file',
+            'schedule': crontab(hour=23, minute=00),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+    }
+    CELERY_QUEUES = []
 
     FROM_NUMBER = 'development'
 
@@ -379,6 +336,10 @@ class Config(object):
 
     HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
 
+    # Format is as follows:
+    # {"dataset_1": "token_1", ...}
+    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
+
     TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
     TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
 
@@ -388,17 +349,9 @@ class Config(object):
     # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
     MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
     FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
+    SES_STUB_URL = os.environ.get("SES_STUB_URL")
 
-    AWS_REGION = 'us-west-2'
-
-    CBC_PROXY_ENABLED = True
-    CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '')
-    CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '')
-
-    ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE}
-
-    # as defined in api db migration 0331_add_broadcast_org.py
-    BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df'
+    AWS_REGION = 'eu-west-1'
 
 
 ######################
@@ -409,10 +362,8 @@ class Development(Config):
     DEBUG = True
     SQLALCHEMY_ECHO = False
 
-    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
-
-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
+    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
+    CONTACT_LIST_BUCKET_NAME = 'development-contact-list'
     TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
     LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
@@ -421,11 +372,7 @@ class Development(Config):
     TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
 
-    INTERNAL_CLIENT_API_KEYS = {
-        Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
-        Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
-    }
-
+    API_INTERNAL_SECRETS = ['dev-notify-secret-key']
     SECRET_KEY = 'dev-notify-secret-key'
     DANGEROUS_SALT = 'dev-notify-salt'
 
@@ -434,18 +381,23 @@ class Development(Config):
 
     NOTIFY_ENVIRONMENT = 'development'
     NOTIFY_LOG_PATH = 'application.log'
-    NOTIFY_EMAIL_DOMAIN = "dispostable.com"
+    NOTIFICATION_QUEUE_PREFIX = 'development'
+    NOTIFY_EMAIL_DOMAIN = "notify.tools"
 
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
-    REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
+    SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
+    REDIS_URL = 'redis://localhost:6379/0'
 
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 
-    API_HOST_NAME = os.getenv('API_HOST_NAME', 'http://localhost:6011')
-    API_RATE_LIMIT_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
 
-    CBC_PROXY_ENABLED = False
+    API_HOST_NAME = "http://localhost:6011"
+    API_RATE_LIMIT_ENABLED = True
+
+    CBC_PROXY_URL = 'http://localhost:8080'
 
 
 class Test(Development):
@@ -472,15 +424,17 @@ class Test(Development):
     LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
 
     # this is overriden in jenkins and on cloudfoundry
-    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
 
-    CELERY = {
-        **Config.CELERY,
-        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
-    }
+    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
 
     ANTIVIRUS_ENABLED = True
 
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
     API_RATE_LIMIT_ENABLED = True
     API_HOST_NAME = "http://localhost:6011"
 
@@ -488,12 +442,11 @@ class Test(Development):
     FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
     TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
 
+    CBC_PROXY_URL = 'http://test-cbc-proxy'
+
     MMG_URL = 'https://example.com/mmg'
     FIRETEXT_URL = 'https://example.com/firetext'
 
-    CBC_PROXY_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
-
 
 class Preview(Config):
     NOTIFY_EMAIL_DOMAIN = 'notify.works'
@@ -507,6 +460,7 @@ class Preview(Config):
     INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
     TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-preview.cloudapps.digital'
     FROM_NUMBER = 'preview'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = False
@@ -524,6 +478,7 @@ class Staging(Config):
     INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
     TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-staging.cloudapps.digital'
     FROM_NUMBER = 'stage'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
@@ -541,7 +496,9 @@ class Live(Config):
     INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
     TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
+    CBC_PROXY_URL = 'https://notify-stub-cbc-production.cloudapps.digital'
     FROM_NUMBER = 'GOVUK'
+    PERFORMANCE_PLATFORM_ENABLED = True
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
     SES_STUB_URL = None

app/cronitor.py

@@ -1,19 +1,15 @@
-from functools import wraps
-
 import requests
+from functools import wraps
 from flask import current_app
 
 
 def cronitor(task_name):
+    # check if task_name is in config
     def decorator(func):
         def ping_cronitor(command):
             if not current_app.config['CRONITOR_ENABLED']:
                 return
 
-            # it's useful to have a log that a periodic task has started in case it
-            # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
-
             task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
             if not task_slug:
                 current_app.logger.error(
@@ -42,11 +38,13 @@ def cronitor(task_name):
         @wraps(func)
         def inner_decorator(*args, **kwargs):
             ping_cronitor('run')
-            status = 'fail'
             try:
                 ret = func(*args, **kwargs)
                 status = 'complete'
                 return ret
+            except Exception:
+                status = 'fail'
+                raise
             finally:
                 ping_cronitor(status)
 

app/dao/__init__.py

@@ -1,5 +1,4 @@
 from sqlalchemy.exc import SQLAlchemyError
-
 from app import db
 
 

app/dao/annual_billing_dao.py

@@ -1,12 +1,10 @@
-from flask import current_app
-
 from app import db
-from app.dao.dao_utils import autocommit
-from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.dao_utils import transactional
 from app.models import AnnualBilling
+from app.dao.date_util import get_current_financial_year_start_year
 
 
-@autocommit
+@transactional
 def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
     result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
@@ -25,7 +23,7 @@ def dao_get_annual_billing(service_id):
     ).order_by(AnnualBilling.financial_year_start).all()
 
 
-@autocommit
+@transactional
 def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
     AnnualBilling.query.filter(
         AnnualBilling.service_id == service_id,
@@ -51,67 +49,3 @@ def dao_get_all_free_sms_fragment_limit(service_id):
     return AnnualBilling.query.filter_by(
         service_id=service_id,
     ).order_by(AnnualBilling.financial_year_start).all()
-
-
-def set_default_free_allowance_for_service(service, year_start=None):
-    default_free_sms_fragment_limits = {
-        'central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_central': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'nhs_local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_gp': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'emergency_service': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'school_or_college': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'other': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-    }
-    if not year_start:
-        year_start = get_current_financial_year_start_year()
-    # handle cases where the year is less than 2020 or greater than 2021
-    if year_start < 2020:
-        year_start = 2020
-    if year_start > 2022:
-        year_start = 2022
-    if service.organisation_type:
-        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
-    else:
-        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
-                                f"{default_free_sms_fragment_limits['other'][year_start]}")
-        free_allowance = default_free_sms_fragment_limits['other'][year_start]
-
-    return dao_create_or_update_annual_billing_for_year(
-        service.id,
-        free_allowance,
-        year_start
-    )

app/dao/api_key_dao.py

@@ -1,14 +1,18 @@
 import uuid
 from datetime import datetime, timedelta
 
-from sqlalchemy import func, or_
-
 from app import db
-from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey
 
+from app.dao.dao_utils import (
+    transactional,
+    version_class
+)
 
-@autocommit
+from sqlalchemy import or_, func
+
+
+@transactional
 @version_class(ApiKey)
 def save_model_api_key(api_key):
     if not api_key.id:
@@ -17,7 +21,7 @@ def save_model_api_key(api_key):
     db.session.add(api_key)
 
 
-@autocommit
+@transactional
 @version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
     api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()

app/dao/broadcast_message_dao.py

@@ -1,20 +1,4 @@
-import uuid
-from datetime import datetime
-
-from sqlalchemy import desc
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    BroadcastEvent,
-    BroadcastMessage,
-    BroadcastProvider,
-    BroadcastProviderMessage,
-    BroadcastProviderMessageNumber,
-    BroadcastProviderMessageStatus,
-    BroadcastStatusType,
-    ServiceBroadcastSettings,
-)
+from app.models import BroadcastMessage, BroadcastEvent
 
 
 def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id):
@@ -24,19 +8,12 @@ def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service
     ).one()
 
 
-def dao_get_broadcast_message_by_references_and_service_id(references_to_original_broadcast, service_id):
-    return BroadcastMessage.query.filter(
-        BroadcastMessage.status.in_((
-            BroadcastStatusType.PENDING_APPROVAL,
-            BroadcastStatusType.BROADCASTING,
-        )),
-        BroadcastMessage.reference.in_(references_to_original_broadcast),
-        BroadcastMessage.service_id == service_id
-    ).one()
+def dao_get_broadcast_message_by_id(broadcast_message_id):
+    return BroadcastMessage.query.get(broadcast_message_id)
 
 
 def dao_get_broadcast_event_by_id(broadcast_event_id):
-    return BroadcastEvent.query.filter(BroadcastEvent.id == broadcast_event_id).one()
+    return BroadcastEvent.query.get(broadcast_event_id)
 
 
 def dao_get_broadcast_messages_for_service(service_id):
@@ -45,27 +22,6 @@ def dao_get_broadcast_messages_for_service(service_id):
     ).order_by(BroadcastMessage.created_at)
 
 
-def dao_get_all_broadcast_messages():
-    return db.session.query(
-        BroadcastMessage.id,
-        BroadcastMessage.reference,
-        ServiceBroadcastSettings.channel,
-        BroadcastMessage.content,
-        BroadcastMessage.areas,
-        BroadcastMessage.status,
-        BroadcastMessage.starts_at,
-        BroadcastMessage.finishes_at,
-        BroadcastMessage.approved_at,
-        BroadcastMessage.cancelled_at,
-    ).join(
-        ServiceBroadcastSettings, ServiceBroadcastSettings.service_id == BroadcastMessage.service_id
-    ).filter(
-        BroadcastMessage.starts_at >= datetime(2021, 5, 25, 0, 0, 0),
-        BroadcastMessage.stubbed == False,  # noqa
-        BroadcastMessage.status.in_(BroadcastStatusType.LIVE_STATUSES)
-    ).order_by(desc(BroadcastMessage.starts_at)).all()
-
-
 def get_earlier_events_for_broadcast_event(broadcast_event_id):
     """
     This is used to build up the references list.
@@ -78,28 +34,3 @@ def get_earlier_events_for_broadcast_event(broadcast_event_id):
     ).order_by(
         BroadcastEvent.sent_at.asc()
     ).all()
-
-
-@autocommit
-def create_broadcast_provider_message(broadcast_event, provider):
-    broadcast_provider_message_id = uuid.uuid4()
-    provider_message = BroadcastProviderMessage(
-        id=broadcast_provider_message_id,
-        broadcast_event=broadcast_event,
-        provider=provider,
-        status=BroadcastProviderMessageStatus.SENDING,
-    )
-    db.session.add(provider_message)
-    db.session.commit()
-    provider_message_number = None
-    if provider == BroadcastProvider.VODAFONE:
-        provider_message_number = BroadcastProviderMessageNumber(
-            broadcast_provider_message_id=broadcast_provider_message_id)
-        db.session.add(provider_message_number)
-        db.session.commit()
-    return provider_message
-
-
-@autocommit
-def update_broadcast_provider_message_status(broadcast_provider_message, *, status):
-    broadcast_provider_message.status = status

app/dao/broadcast_service_dao.py

@@ -1,100 +0,0 @@
-from datetime import datetime
-
-from flask import current_app
-
-from app import db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    BROADCAST_TYPE,
-    EMAIL_AUTH_TYPE,
-    INVITE_PENDING,
-    VIEW_ACTIVITY,
-    ApiKey,
-    InvitedUser,
-    Organisation,
-    Permission,
-    Service,
-    ServiceBroadcastSettings,
-    ServicePermission,
-)
-
-
-@autocommit
-@version_class(Service)
-def set_broadcast_service_type(service, service_mode, broadcast_channel, provider_restriction):
-    insert_or_update_service_broadcast_settings(
-        service, channel=broadcast_channel, provider_restriction=provider_restriction
-    )
-
-    # Remove all permissions and add broadcast permission
-    if not service.has_permission(BROADCAST_TYPE):
-        service_permission = ServicePermission(service_id=service.id, permission=BROADCAST_TYPE)
-        db.session.add(service_permission)
-
-    ServicePermission.query.filter(
-        ServicePermission.service_id == service.id,
-        ServicePermission.permission != BROADCAST_TYPE,
-        # Email auth is an exception to the other service permissions (which relate to what type
-        # of notifications a service can send) where a broadcast service is allowed to have the
-        # email auth permission (but doesn't have to)
-        ServicePermission.permission != EMAIL_AUTH_TYPE
-    ).delete()
-
-    # Refresh the service object as it has references to the service permissions but we don't yet
-    # want to commit the permission changes incase all of this needs to rollback
-    db.session.refresh(service)
-
-    # Set service count as live false always
-    service.count_as_live = False
-
-    # Set service into training mode or live mode
-    if service_mode == "live":
-        if service.restricted:
-            # Only update the go live at timestamp if this if moving from training mode
-            # to live mode, not if it's moving from one type of live mode service to another
-            service.go_live_at = datetime.utcnow()
-        service.restricted = False
-    else:
-        service.restricted = True
-        service.go_live_at = None
-
-    # Remove all user permissions apart from view_activity for the service users and invited users
-    Permission.query.filter(
-        Permission.service_id == service.id,
-        Permission.permission != VIEW_ACTIVITY
-    ).delete()
-    InvitedUser.query.filter_by(
-        service_id=service.id,
-        status=INVITE_PENDING
-    ).update({'permissions': VIEW_ACTIVITY})
-
-    # Revoke any API keys to avoid a regular API key being used to send alerts
-    ApiKey.query.filter_by(
-        service_id=service.id,
-        expiry_date=None,
-    ).update({
-        ApiKey.expiry_date: datetime.utcnow()
-    })
-
-    # Add service to organisation
-    organisation = Organisation.query.filter_by(
-        id=current_app.config['BROADCAST_ORGANISATION_ID']
-    ).one()
-    service.organisation_id = organisation.id
-    service.organisation_type = organisation.organisation_type
-    service.crown = organisation.crown
-
-    db.session.add(service)
-
-
-def insert_or_update_service_broadcast_settings(service, channel, provider_restriction="all"):
-    if not service.service_broadcast_settings:
-        settings = ServiceBroadcastSettings()
-        settings.service = service
-        settings.channel = channel
-        settings.provider = provider_restriction
-        db.session.add(settings)
-    else:
-        service.service_broadcast_settings.channel = channel
-        service.service_broadcast_settings.provider = provider_restriction
-        db.session.add(service.service_broadcast_settings)

app/dao/complaint_dao.py

@@ -4,12 +4,12 @@ from flask import current_app
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import Complaint
 from app.utils import get_london_midnight_in_utc
 
 
-@autocommit
+@transactional
 def save_complaint(complaint):
     db.session.add(complaint)
 

app/dao/daily_sorted_letter_dao.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import DailySortedLetter
 
 
@@ -13,7 +13,7 @@ def dao_get_daily_sorted_letter_by_billing_day(billing_day):
     ).first()
 
 
-@autocommit
+@transactional
 def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
     '''
     This uses the Postgres upsert to avoid race conditions when two threads try and insert

app/dao/dao_utils.py

@@ -1,20 +1,16 @@
 import itertools
-from contextlib import contextmanager
 from functools import wraps
 
 from app import db
 from app.history_meta import create_history
 
 
-def autocommit(func):
+def transactional(func):
     @wraps(func)
     def commit_or_rollback(*args, **kwargs):
         try:
             res = func(*args, **kwargs)
-
-            if not db.session.registry().transaction.nested:
-                db.session.commit()
-
+            db.session.commit()
             return res
         except Exception:
             db.session.rollback()
@@ -22,20 +18,6 @@ def autocommit(func):
     return commit_or_rollback
 
 
-@contextmanager
-def transaction():
-    try:
-        db.session.begin_nested()
-        yield
-        db.session.commit()
-
-        if not db.session.registry().transaction.nested:
-            db.session.commit()
-    except Exception:
-        db.session.rollback()
-        raise
-
-
 class VersionOptions():
 
     def __init__(self, model_class, history_class=None, must_write_history=True):
@@ -93,7 +75,7 @@ def dao_rollback():
     db.session.rollback()
 
 
-@autocommit
+@transactional
 def dao_save_object(obj):
     # add/update object in db
     db.session.add(obj)

app/dao/date_util.py

@@ -1,7 +1,7 @@
-from datetime import date, datetime, time, timedelta
+from datetime import datetime, timedelta, date, time
 
+from notifications_utils.timezones import convert_bst_to_utc
 import pytz
-from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 
 
 def get_months_for_financial_year(year):
@@ -22,15 +22,6 @@ def get_financial_year(year):
     return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
 
 
-def get_financial_year_dates(year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    return (
-        convert_utc_to_bst(year_start_datetime).date(),
-        convert_utc_to_bst(year_end_datetime).date()
-    )
-
-
 def get_current_financial_year():
     now = datetime.utcnow()
     current_month = int(now.strftime('%-m'))

app/dao/email_branding_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import EmailBranding
 
 
@@ -15,12 +15,12 @@ def dao_get_email_branding_by_name(email_branding_name):
     return EmailBranding.query.filter_by(name=email_branding_name).first()
 
 
-@autocommit
+@transactional
 def dao_create_email_branding(email_branding):
     db.session.add(email_branding)
 
 
-@autocommit
+@transactional
 def dao_update_email_branding(email_branding, **kwargs):
     for key, value in kwargs.items():
         setattr(email_branding, key, value or None)

app/dao/fact_billing_dao.py

@@ -1,42 +1,41 @@
-from datetime import date, datetime, timedelta
+from datetime import datetime, timedelta, time, date
 
 from flask import current_app
-from notifications_utils.timezones import convert_utc_to_bst
-from sqlalchemy import Date, Integer, and_, desc, func, union
+from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import func, desc, Date, Integer, and_
 from sqlalchemy.sql.expression import case, literal
 
 from app import db
 from app.dao.date_util import (
-    get_financial_year_dates,
-    get_financial_year_for_datetime,
+    get_financial_year,
+    get_financial_year_for_datetime
 )
 from app.dao.organisation_dao import dao_get_organisation_live_services
+
 from app.models import (
-    EMAIL_TYPE,
-    INTERNATIONAL_POSTAGE_TYPES,
-    KEY_TYPE_NORMAL,
-    KEY_TYPE_TEAM,
+    FactBilling,
+    Service,
+    KEY_TYPE_TEST,
     LETTER_TYPE,
-    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS,
+    SMS_TYPE,
+    Rate,
+    LetterRate,
+    NotificationHistory,
+    EMAIL_TYPE,
     NOTIFICATION_STATUS_TYPES_BILLABLE_SMS,
     NOTIFICATION_STATUS_TYPES_SENT_EMAILS,
-    SMS_TYPE,
+    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS,
     AnnualBilling,
-    FactBilling,
-    LetterRate,
-    NotificationAllTimeView,
-    NotificationHistory,
     Organisation,
-    Rate,
-    Service,
+    INTERNATIONAL_POSTAGE_TYPES,
 )
-from app.utils import get_london_midnight_in_utc
+from app.utils import get_london_midnight_in_utc, get_notification_table_to_use
 
 
-def fetch_sms_free_allowance_remainder_until_date(end_date):
+def fetch_sms_free_allowance_remainder(start_date):
     # ASSUMPTION: AnnualBilling has been populated for year.
-    billing_year = get_financial_year_for_datetime(end_date)
+    billing_year = get_financial_year_for_datetime(start_date)
     start_of_year = date(billing_year, 4, 1)
 
     billable_units = func.coalesce(func.sum(FactBilling.billable_units * FactBilling.rate_multiplier), 0)
@@ -52,7 +51,7 @@ def fetch_sms_free_allowance_remainder_until_date(end_date):
         FactBilling, and_(
             AnnualBilling.service_id == FactBilling.service_id,
             FactBilling.bst_date >= start_of_year,
-            FactBilling.bst_date < end_date,
+            FactBilling.bst_date < start_date,
             FactBilling.notification_type == SMS_TYPE,
         )
     ).filter(
@@ -67,18 +66,14 @@ def fetch_sms_free_allowance_remainder_until_date(end_date):
 def fetch_sms_billing_for_all_services(start_date, end_date):
 
     # ASSUMPTION: AnnualBilling has been populated for year.
-    allowance_left_at_start_date_query = fetch_sms_free_allowance_remainder_until_date(start_date).subquery()
+    free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery()
 
     sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
-
-    # subtract sms_billable_units units accrued since report's start date to get up-to-date
-    # allowance remainder
-    sms_allowance_left = func.greatest(allowance_left_at_start_date_query.c.sms_remainder - sms_billable_units, 0)
-
-    # billable units here are for period between start date and end date only, so to see
-    # how many are chargeable, we need to see how much free allowance was used up in the
-    # period up until report's start date and then do a subtraction
-    chargeable_sms = func.greatest(sms_billable_units - allowance_left_at_start_date_query.c.sms_remainder, 0)
+    sms_remainder = func.coalesce(
+        free_allowance_remainder.c.sms_remainder,
+        free_allowance_remainder.c.free_sms_fragment_limit
+    )
+    chargeable_sms = func.greatest(sms_billable_units - sms_remainder, 0)
     sms_cost = chargeable_sms * FactBilling.rate
 
     query = db.session.query(
@@ -86,16 +81,16 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
         Organisation.id.label('organisation_id'),
         Service.name.label("service_name"),
         Service.id.label("service_id"),
-        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
+        free_allowance_remainder.c.free_sms_fragment_limit,
         FactBilling.rate.label('sms_rate'),
-        sms_allowance_left.label("sms_remainder"),
+        sms_remainder.label("sms_remainder"),
         sms_billable_units.label('sms_billable_units'),
         chargeable_sms.label("chargeable_billable_sms"),
         sms_cost.label('sms_cost'),
     ).select_from(
         Service
     ).outerjoin(
-        allowance_left_at_start_date_query, Service.id == allowance_left_at_start_date_query.c.service_id
+        free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id
     ).outerjoin(
         Service.organisation
     ).join(
@@ -109,8 +104,8 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
         Organisation.id,
         Service.id,
         Service.name,
-        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
-        allowance_left_at_start_date_query.c.sms_remainder,
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        free_allowance_remainder.c.sms_remainder,
         FactBilling.rate,
     ).order_by(
         Organisation.name,
@@ -120,13 +115,12 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
     return query.all()
 
 
-def fetch_letter_costs_and_totals_for_all_services(start_date, end_date):
+def fetch_letter_costs_for_all_services(start_date, end_date):
     query = db.session.query(
         Organisation.name.label("organisation_name"),
         Organisation.id.label("organisation_id"),
         Service.name.label("service_name"),
         Service.id.label("service_id"),
-        func.sum(FactBilling.notifications_sent).label("total_letters"),
         func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
     ).select_from(
         Service
@@ -157,12 +151,9 @@ def fetch_letter_line_items_for_all_services(start_date, end_date):
         [(FactBilling.postage.in_(INTERNATIONAL_POSTAGE_TYPES), "international")], else_=FactBilling.postage
     ).label("postage")
 
-    postage_order = case(
-            (formatted_postage == "second", 1),
-            (formatted_postage == "first", 2),
-            (formatted_postage == "international", 3),
-            else_=0  # assumes never get 0 as a result
-    )
+    postage_order = case(((formatted_postage == "second", 1),
+                          (formatted_postage == "first", 2),
+                          (formatted_postage == "international", 3)))
 
     query = db.session.query(
         Organisation.name.label("organisation_name"),
@@ -199,235 +190,111 @@ def fetch_letter_line_items_for_all_services(start_date, end_date):
 
 
 def fetch_billing_totals_for_year(service_id, year):
+    year_start_date, year_end_date = get_financial_year(year)
     """
-    Returns a row for each distinct rate and notification_type from ft_billing
-    over the specified financial year e.g.
-
-        (
-            rate=0.0165,
-            notification_type=sms,
-            notifications_sent=123,
-            ...
-        )
-
-    The "query_service_<type>..." subqueries for each notification_type all
-    return the same columns but differ internally e.g. SMS has to incorporate
-    a rate multiplier. Each subquery returns the same set of columns, which we
-    pick from here before the big union.
+      Billing for email: only record the total number of emails.
+      Billing for letters: The billing units is used to fetch the correct rate for the sheet count of the letter.
+      Total cost is notifications_sent * rate.
+      Rate multiplier does not apply to email or letters.
     """
-    return db.session.query(
-        union(*[
-            db.session.query(
-                query.c.notification_type.label("notification_type"),
-                query.c.rate.label("rate"),
+    email_and_letters = db.session.query(
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.notifications_sent).label("billable_units"),
+        FactBilling.rate.label('rate'),
+        FactBilling.notification_type.label('notification_type')
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
+    ).group_by(
+        FactBilling.rate,
+        FactBilling.notification_type
+    )
+    """
+    Billing for SMS using the billing_units * rate_multiplier. Billing unit of SMS is the fragment count of a message
+    """
+    sms = db.session.query(
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
+        FactBilling.rate,
+        FactBilling.notification_type
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type == SMS_TYPE
+    ).group_by(
+        FactBilling.rate,
+        FactBilling.notification_type
+    )
 
-                func.sum(query.c.notifications_sent).label("notifications_sent"),
-                func.sum(query.c.chargeable_units).label("chargeable_units"),
-                func.sum(query.c.cost).label("cost"),
-                func.sum(query.c.free_allowance_used).label("free_allowance_used"),
-                func.sum(query.c.charged_units).label("charged_units"),
-            ).group_by(
-                query.c.rate,
-                query.c.notification_type
-            )
-            for query in [
-                query_service_sms_usage_for_year(service_id, year).subquery(),
-                query_service_email_usage_for_year(service_id, year).subquery(),
-                query_service_letter_usage_for_year(service_id, year).subquery(),
-            ]
-        ]).subquery()
-    ).order_by(
-        "notification_type",
-        "rate",
+    yearly_data = email_and_letters.union_all(sms).order_by(
+        'notification_type',
+        'rate'
     ).all()
 
+    return yearly_data
+
 
 def fetch_monthly_billing_for_year(service_id, year):
-    """
-    Returns a row for each distinct rate, notification_type, postage and month
-    from ft_billing over the specified financial year e.g.
+    year_start_datetime, year_end_datetime = get_financial_year(year)
 
-        (
-            rate=0.0165,
-            notification_type=sms,
-            postage=none,
-            month=2022-04-01 00:00:00,
-            notifications_sent=123,
-            ...
-        )
+    year_start_date = convert_utc_to_bst(year_start_datetime).date()
+    year_end_date = convert_utc_to_bst(year_end_datetime).date()
 
-    The "postage" field is "none" except for letters. Each subquery takes care
-    of anything specific to the notification type e.g. rate multipliers for SMS.
-
-    Since the data in ft_billing is only refreshed once a day for all services,
-    we also update the table on-the-fly if we need accurate data for this year.
-    """
-    _, year_end = get_financial_year_dates(year)
     today = convert_utc_to_bst(datetime.utcnow()).date()
-
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end >= today:
+    if year_end_date >= today:
         data = fetch_billing_data_for_day(process_day=today, service_id=service_id, check_permissions=True)
         for d in data:
             update_fact_billing(data=d, process_day=today)
 
-    return db.session.query(
-        union(*[
-            db.session.query(
-                query.c.rate.label("rate"),
-                query.c.notification_type.label("notification_type"),
-                query.c.postage.label("postage"),
-                func.date_trunc('month', query.c.bst_date).cast(Date).label("month"),
+    email_and_letters = db.session.query(
+        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.notifications_sent).label("billable_units"),
+        FactBilling.rate.label('rate'),
+        FactBilling.notification_type.label('notification_type'),
+        FactBilling.postage
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
+    ).group_by(
+        'month',
+        FactBilling.rate,
+        FactBilling.notification_type,
+        FactBilling.postage
+    )
 
-                func.sum(query.c.notifications_sent).label("notifications_sent"),
-                func.sum(query.c.chargeable_units).label("chargeable_units"),
-                func.sum(query.c.cost).label("cost"),
-                func.sum(query.c.free_allowance_used).label("free_allowance_used"),
-                func.sum(query.c.charged_units).label("charged_units"),
-            ).group_by(
-                query.c.rate,
-                query.c.notification_type,
-                query.c.postage,
-                'month',
-            )
-            for query in [
-                query_service_sms_usage_for_year(service_id, year).subquery(),
-                query_service_email_usage_for_year(service_id, year).subquery(),
-                query_service_letter_usage_for_year(service_id, year).subquery(),
-            ]
-        ]).subquery()
-    ).order_by(
-        "month",
-        "notification_type",
-        "rate",
+    sms = db.session.query(
+        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
+        FactBilling.rate,
+        FactBilling.notification_type,
+        FactBilling.postage
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type == SMS_TYPE
+    ).group_by(
+        'month',
+        FactBilling.rate,
+        FactBilling.notification_type,
+        FactBilling.postage
+    )
+
+    yearly_data = email_and_letters.union_all(sms).order_by(
+        'month',
+        'notification_type',
+        'rate'
     ).all()
 
-
-def query_service_email_usage_for_year(service_id, year):
-    year_start, year_end = get_financial_year_dates(year)
-
-    return db.session.query(
-        FactBilling.bst_date,
-        FactBilling.postage,  # should always be "none"
-        FactBilling.notifications_sent,
-        FactBilling.billable_units.label("chargeable_units"),
-        FactBilling.rate,
-        FactBilling.notification_type,
-        literal(0).label("cost"),
-        literal(0).label("free_allowance_used"),
-        FactBilling.billable_units.label("charged_units"),
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start,
-        FactBilling.bst_date <= year_end,
-        FactBilling.notification_type == EMAIL_TYPE
-    )
-
-
-def query_service_letter_usage_for_year(service_id, year):
-    year_start, year_end = get_financial_year_dates(year)
-
-    return db.session.query(
-        FactBilling.bst_date,
-        FactBilling.postage,
-        FactBilling.notifications_sent,
-        # We can't use billable_units here as it represents the
-        # sheet count for letters, which is already accounted for
-        # in the rate. We actually charge per letter, not sheet.
-        FactBilling.notifications_sent.label("chargeable_units"),
-        FactBilling.rate,
-        FactBilling.notification_type,
-        (FactBilling.notifications_sent * FactBilling.rate).label("cost"),
-        literal(0).label("free_allowance_used"),
-        FactBilling.notifications_sent.label("charged_units"),
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start,
-        FactBilling.bst_date <= year_end,
-        FactBilling.notification_type == LETTER_TYPE
-    )
-
-
-def query_service_sms_usage_for_year(service_id, year):
-    """
-    Returns rows from the ft_billing table with some calculated values like cost,
-    incorporating the SMS free allowance e.g.
-
-        (
-            bst_date=2022-04-27,
-            notifications_sent=12,
-            chargeable_units=12,
-            rate=0.0165,
-            [cost=0      <== covered by the free allowance],
-            [cost=0.198  <== if free allowance exhausted],
-            [cost=0.099  <== only some free allowance left],
-            ...
-        )
-
-    In order to calculate how much free allowance is left, we need to work out
-    how much was used for previous bst_dates - cumulative_chargeable_units -
-    which we then subtract from the free allowance for the year.
-
-    cumulative_chargeable_units is calculated using a "window" clause, which has
-    access to all the rows identified by the query filter. Note that it's not
-    affected by any GROUP BY clauses that happen in outer queries.
-
-    https://www.postgresql.org/docs/current/tutorial-window.html
-
-    ASSUMPTION: rates always change at midnight i.e. there can only be one rate
-    on a given bst_date. This means we don't need to worry about how to assign
-    free allowance if it happens to run out when a rate changes.
-    """
-    year_start, year_end = get_financial_year_dates(year)
-    this_rows_chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
-
-    # Subquery for the number of chargeable units in all rows preceding this one,
-    # which might be none if this is the first row (hence the "coalesce"). For
-    # some reason the end result is a decimal despite all the input columns being
-    # integer - this seems to be a Sqlalchemy quirk (works in raw SQL).
-    chargeable_units_used_before_this_row = func.coalesce(
-        func.sum(this_rows_chargeable_units).over(
-            # order is "ASC" by default
-            order_by=[FactBilling.bst_date],
-            # first row to previous row
-            rows=(None, -1)
-        ).cast(Integer),
-        0
-    )
-
-    # Subquery for how much free allowance we have left before the current row,
-    # so we can work out the cost for this row after taking it into account.
-    remaining_free_allowance_before_this_row = func.greatest(
-        AnnualBilling.free_sms_fragment_limit - chargeable_units_used_before_this_row,
-        0
-    )
-
-    # Subquery for the number of chargeable_units that we will actually charge
-    # for, after taking any remaining free allowance into account.
-    charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
-
-    free_allowance_used = func.least(remaining_free_allowance_before_this_row, this_rows_chargeable_units)
-
-    return db.session.query(
-        FactBilling.bst_date,
-        FactBilling.postage,  # should always be "none"
-        FactBilling.notifications_sent,
-        this_rows_chargeable_units.label("chargeable_units"),
-        FactBilling.rate,
-        FactBilling.notification_type,
-        (charged_units * FactBilling.rate).label("cost"),
-        free_allowance_used.label("free_allowance_used"),
-        charged_units.label("charged_units"),
-    ).join(
-        AnnualBilling,
-        AnnualBilling.service_id == service_id
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start,
-        FactBilling.bst_date <= year_end,
-        FactBilling.notification_type == SMS_TYPE,
-        AnnualBilling.financial_year_start == year,
-    )
+    return yearly_data
 
 
 def delete_billing_data_for_service_for_day(process_day, service_id):
@@ -443,8 +310,8 @@ def delete_billing_data_for_service_for_day(process_day, service_id):
 
 
 def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False):
-    start_date = get_london_midnight_in_utc(process_day)
-    end_date = get_london_midnight_in_utc(process_day + timedelta(days=1))
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
     current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date))
     transit_data = []
     if not service_id:
@@ -455,7 +322,10 @@ def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=F
     for service in services:
         for notification_type in (SMS_TYPE, EMAIL_TYPE, LETTER_TYPE):
             if (not check_permissions) or service.has_permission(notification_type):
+                table = get_notification_table_to_use(service, notification_type, process_day,
+                                                      has_delete_task_run=False)
                 results = _query_for_billing_data(
+                    table=table,
                     notification_type=notification_type,
                     start_date=start_date,
                     end_date=end_date,
@@ -466,10 +336,10 @@ def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=F
     return transit_data
 
 
-def _query_for_billing_data(notification_type, start_date, end_date, service):
+def _query_for_billing_data(table, notification_type, start_date, end_date, service):
     def _email_query():
         return db.session.query(
-            NotificationAllTimeView.template_id,
+            table.template_id,
             literal(service.crown).label('crown'),
             literal(service.id).label('service_id'),
             literal(notification_type).label('notification_type'),
@@ -481,22 +351,22 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
             literal(0).label('billable_units'),
             func.count().label('notifications_sent'),
         ).filter(
-            NotificationAllTimeView.status.in_(NOTIFICATION_STATUS_TYPES_SENT_EMAILS),
-            NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.notification_type == notification_type,
-            NotificationAllTimeView.service_id == service.id
+            table.status.in_(NOTIFICATION_STATUS_TYPES_SENT_EMAILS),
+            table.key_type != KEY_TYPE_TEST,
+            table.created_at >= start_date,
+            table.created_at < end_date,
+            table.notification_type == notification_type,
+            table.service_id == service.id
         ).group_by(
-            NotificationAllTimeView.template_id,
+            table.template_id,
         )
 
     def _sms_query():
-        sent_by = func.coalesce(NotificationAllTimeView.sent_by, 'unknown')
-        rate_multiplier = func.coalesce(NotificationAllTimeView.rate_multiplier, 1).cast(Integer)
-        international = func.coalesce(NotificationAllTimeView.international, False)
+        sent_by = func.coalesce(table.sent_by, 'unknown')
+        rate_multiplier = func.coalesce(table.rate_multiplier, 1).cast(Integer)
+        international = func.coalesce(table.international, False)
         return db.session.query(
-            NotificationAllTimeView.template_id,
+            table.template_id,
             literal(service.crown).label('crown'),
             literal(service.id).label('service_id'),
             literal(notification_type).label('notification_type'),
@@ -505,50 +375,50 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
             international.label('international'),
             literal(None).label('letter_page_count'),
             literal('none').label('postage'),
-            func.sum(NotificationAllTimeView.billable_units).label('billable_units'),
+            func.sum(table.billable_units).label('billable_units'),
             func.count().label('notifications_sent'),
         ).filter(
-            NotificationAllTimeView.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_SMS),
-            NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.notification_type == notification_type,
-            NotificationAllTimeView.service_id == service.id
+            table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_SMS),
+            table.key_type != KEY_TYPE_TEST,
+            table.created_at >= start_date,
+            table.created_at < end_date,
+            table.notification_type == notification_type,
+            table.service_id == service.id
         ).group_by(
-            NotificationAllTimeView.template_id,
+            table.template_id,
             sent_by,
             rate_multiplier,
             international,
         )
 
     def _letter_query():
-        rate_multiplier = func.coalesce(NotificationAllTimeView.rate_multiplier, 1).cast(Integer)
-        postage = func.coalesce(NotificationAllTimeView.postage, 'none')
+        rate_multiplier = func.coalesce(table.rate_multiplier, 1).cast(Integer)
+        postage = func.coalesce(table.postage, 'none')
         return db.session.query(
-            NotificationAllTimeView.template_id,
+            table.template_id,
             literal(service.crown).label('crown'),
             literal(service.id).label('service_id'),
             literal(notification_type).label('notification_type'),
             literal('dvla').label('sent_by'),
             rate_multiplier.label('rate_multiplier'),
-            NotificationAllTimeView.international,
-            NotificationAllTimeView.billable_units.label('letter_page_count'),
+            table.international,
+            table.billable_units.label('letter_page_count'),
             postage.label('postage'),
-            func.sum(NotificationAllTimeView.billable_units).label('billable_units'),
+            func.sum(table.billable_units).label('billable_units'),
             func.count().label('notifications_sent'),
         ).filter(
-            NotificationAllTimeView.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS),
-            NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
-            NotificationAllTimeView.created_at >= start_date,
-            NotificationAllTimeView.created_at < end_date,
-            NotificationAllTimeView.notification_type == notification_type,
-            NotificationAllTimeView.service_id == service.id
+            table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS),
+            table.key_type != KEY_TYPE_TEST,
+            table.created_at >= start_date,
+            table.created_at < end_date,
+            table.notification_type == notification_type,
+            table.service_id == service.id
         ).group_by(
-            NotificationAllTimeView.template_id,
+            table.template_id,
             rate_multiplier,
-            NotificationAllTimeView.billable_units,
+            table.billable_units,
             postage,
-            NotificationAllTimeView.international
+            table.international
         )
 
     query_funcs = {
@@ -586,8 +456,6 @@ def get_rate(
     if notification_type == LETTER_TYPE:
         if letter_page_count == 0:
             return 0
-        # if crown is not set default to true, this is okay because the rates are the same for both crown and non-crown.
-        crown = crown or True
         return next(
             r.rate
             for r in letter_rates if (
@@ -718,42 +586,45 @@ def fetch_email_usage_for_organisation(organisation_id, start_date, end_date):
     return query.all()
 
 
-def fetch_sms_billing_for_organisation(organisation_id, financial_year):
+def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
     # ASSUMPTION: AnnualBilling has been populated for year.
-    ft_billing_subquery = query_organisation_sms_usage_for_year(organisation_id, financial_year).subquery()
+    free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery()
 
-    sms_billable_units = func.sum(func.coalesce(ft_billing_subquery.c.chargeable_units, 0))
-
-    # subtract sms_billable_units units accrued since report's start date to get up-to-date
-    # allowance remainder
-    sms_allowance_left = func.greatest(AnnualBilling.free_sms_fragment_limit - sms_billable_units, 0)
-
-    chargeable_sms = func.sum(ft_billing_subquery.c.charged_units)
-    sms_cost = func.sum(ft_billing_subquery.c.cost)
+    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+    sms_remainder = func.coalesce(
+        free_allowance_remainder.c.sms_remainder,
+        free_allowance_remainder.c.free_sms_fragment_limit
+    )
+    chargeable_sms = func.greatest(sms_billable_units - sms_remainder, 0)
+    sms_cost = chargeable_sms * FactBilling.rate
 
     query = db.session.query(
         Service.name.label("service_name"),
         Service.id.label("service_id"),
-        AnnualBilling.free_sms_fragment_limit,
-        func.coalesce(sms_allowance_left, 0).label("sms_remainder"),
-        func.coalesce(sms_billable_units, 0).label('sms_billable_units'),
-        func.coalesce(chargeable_sms, 0).label("chargeable_billable_sms"),
-        func.coalesce(sms_cost, 0).label('sms_cost'),
-        Service.active
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        FactBilling.rate.label('sms_rate'),
+        sms_remainder.label("sms_remainder"),
+        sms_billable_units.label('sms_billable_units'),
+        chargeable_sms.label("chargeable_billable_sms"),
+        sms_cost.label('sms_cost'),
     ).select_from(
         Service
     ).outerjoin(
-        AnnualBilling,
-        and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == financial_year)
-    ).outerjoin(
-        ft_billing_subquery, Service.id == ft_billing_subquery.c.service_id
+        free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
     ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == SMS_TYPE,
         Service.organisation_id == organisation_id,
         Service.restricted.is_(False)
     ).group_by(
         Service.id,
         Service.name,
-        AnnualBilling.free_sms_fragment_limit
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        free_allowance_remainder.c.sms_remainder,
+        FactBilling.rate,
     ).order_by(
         Service.name
     )
@@ -761,70 +632,16 @@ def fetch_sms_billing_for_organisation(organisation_id, financial_year):
     return query.all()
 
 
-def query_organisation_sms_usage_for_year(organisation_id, year):
-    """
-    See docstring for query_service_sms_usage_for_year()
-    """
-    year_start, year_end = get_financial_year_dates(year)
-    this_rows_chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
-
-    # Subquery for the number of chargeable units in all rows preceding this one,
-    # which might be none if this is the first row (hence the "coalesce").
-    chargeable_units_used_before_this_row = func.coalesce(
-        func.sum(this_rows_chargeable_units).over(
-            # order is "ASC" by default
-            order_by=[FactBilling.bst_date],
-
-            # partition by service id
-            partition_by=FactBilling.service_id,
-
-            # first row to previous row
-            rows=(None, -1)
-        ).cast(Integer),
-        0
-    )
-
-    # Subquery for how much free allowance we have left before the current row,
-    # so we can work out the cost for this row after taking it into account.
-    remaining_free_allowance_before_this_row = func.greatest(
-        AnnualBilling.free_sms_fragment_limit - chargeable_units_used_before_this_row,
-        0
-    )
-
-    # Subquery for the number of chargeable_units that we will actually charge
-    # for, after taking any remaining free allowance into account.
-    charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
-
-    return db.session.query(
-        Service.id.label('service_id'),
-        FactBilling.bst_date,
-        this_rows_chargeable_units.label("chargeable_units"),
-        (charged_units * FactBilling.rate).label("cost"),
-        charged_units.label("charged_units"),
-    ).join(
-        AnnualBilling,
-        AnnualBilling.service_id == Service.id
-    ).outerjoin(
-        FactBilling,
-        and_(
-            Service.id == FactBilling.service_id,
-            FactBilling.bst_date >= year_start,
-            FactBilling.bst_date <= year_end,
-            FactBilling.notification_type == SMS_TYPE,
-        )
-    ).filter(
-        Service.organisation_id == organisation_id,
-        AnnualBilling.financial_year_start == year,
-    )
-
-
 def fetch_usage_year_for_organisation(organisation_id, year):
-    year_start, year_end = get_financial_year_dates(year)
+    year_start_datetime, year_end_datetime = get_financial_year(year)
+
+    year_start_date = convert_utc_to_bst(year_start_datetime).date()
+    year_end_date = convert_utc_to_bst(year_end_datetime).date()
+
     today = convert_utc_to_bst(datetime.utcnow()).date()
     services = dao_get_organisation_live_services(organisation_id)
-
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end >= today:
+    if year_end_date >= today:
         for service in services:
             data = fetch_billing_data_for_day(process_day=today, service_id=service.id)
             for d in data:
@@ -841,12 +658,11 @@ def fetch_usage_year_for_organisation(organisation_id, year):
             'chargeable_billable_sms': 0,
             'sms_cost': 0.0,
             'letter_cost': 0.0,
-            'emails_sent': 0,
-            'active': service.active
+            'emails_sent': 0
         }
-    sms_usages = fetch_sms_billing_for_organisation(organisation_id, year)
-    letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start, year_end)
-    email_usages = fetch_email_usage_for_organisation(organisation_id, year_start, year_end)
+    sms_usages = fetch_sms_billing_for_organisation(organisation_id, year_start_date, year_end_date)
+    letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start_date, year_end_date)
+    email_usages = fetch_email_usage_for_organisation(organisation_id, year_start_date, year_end_date)
     for usage in sms_usages:
         service_with_usage[str(usage.service_id)] = {
             'service_id': usage.service_id,
@@ -857,8 +673,7 @@ def fetch_usage_year_for_organisation(organisation_id, year):
             'chargeable_billable_sms': usage.chargeable_billable_sms,
             'sms_cost': float(usage.sms_cost),
             'letter_cost': 0.0,
-            'emails_sent': 0,
-            'active': usage.active
+            'emails_sent': 0
         }
     for letter_usage in letter_usages:
         service_with_usage[str(letter_usage.service_id)]['letter_cost'] = float(letter_usage.letter_cost)
@@ -866,193 +681,3 @@ def fetch_usage_year_for_organisation(organisation_id, year):
         service_with_usage[str(email_usage.service_id)]['emails_sent'] = email_usage.emails_sent
 
     return service_with_usage
-
-
-def fetch_billing_details_for_all_services():
-    billing_details = db.session.query(
-        Service.id.label('service_id'),
-        func.coalesce(Service.purchase_order_number, Organisation.purchase_order_number).label('purchase_order_number'),
-        func.coalesce(Service.billing_contact_names, Organisation.billing_contact_names).label('billing_contact_names'),
-        func.coalesce(
-            Service.billing_contact_email_addresses,
-            Organisation.billing_contact_email_addresses
-        ).label('billing_contact_email_addresses'),
-        func.coalesce(Service.billing_reference, Organisation.billing_reference).label('billing_reference'),
-    ).outerjoin(
-        Service.organisation
-    ).all()
-
-    return billing_details
-
-
-def fetch_daily_volumes_for_platform(start_date, end_date):
-    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
-
-    daily_volume_stats = db.session.query(
-        FactBilling.bst_date,
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('sms_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('sms_fragment_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
-            ], else_=0
-        )).label('sms_fragments_times_multiplier'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('email_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
-            ], else_=0
-        )).label('letter_totals'),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('letter_sheet_totals')
-    ).filter(
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).group_by(
-        FactBilling.bst_date,
-        FactBilling.notification_type
-    ).subquery()
-
-    aggregated_totals = db.session.query(
-        daily_volume_stats.c.bst_date.cast(db.Text).label('bst_date'),
-        func.sum(daily_volume_stats.c.sms_totals).label('sms_totals'),
-        func.sum(daily_volume_stats.c.sms_fragment_totals).label('sms_fragment_totals'),
-        func.sum(
-            daily_volume_stats.c.sms_fragments_times_multiplier).label('sms_chargeable_units'),
-        func.sum(daily_volume_stats.c.email_totals).label('email_totals'),
-        func.sum(daily_volume_stats.c.letter_totals).label('letter_totals'),
-        func.sum(daily_volume_stats.c.letter_sheet_totals).label('letter_sheet_totals')
-    ).group_by(
-        daily_volume_stats.c.bst_date
-    ).order_by(
-        daily_volume_stats.c.bst_date
-    ).all()
-
-    return aggregated_totals
-
-
-def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
-    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
-
-    daily_volume_stats = db.session.query(
-        FactBilling.bst_date,
-        FactBilling.provider,
-        func.sum(FactBilling.notifications_sent).label('sms_totals'),
-        func.sum(FactBilling.billable_units).label('sms_fragment_totals'),
-        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('sms_chargeable_units'),
-        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier * FactBilling.rate).label('sms_cost'),
-    ).filter(
-        FactBilling.notification_type == SMS_TYPE,
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date,
-    ).group_by(
-        FactBilling.bst_date,
-        FactBilling.provider,
-    ).order_by(
-        FactBilling.bst_date,
-        FactBilling.provider,
-    ).all()
-
-    return daily_volume_stats
-
-
-def fetch_volumes_by_service(start_date, end_date):
-    # query to return the volume totals by service aggregated for the date range given
-    # start and end dates are inclusive.
-    year_end_date = int(end_date.strftime('%Y'))
-
-    volume_stats = db.session.query(
-        FactBilling.bst_date,
-        FactBilling.service_id,
-        func.sum(case([
-            (FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('sms_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == SMS_TYPE, FactBilling.billable_units * FactBilling.rate_multiplier)
-        ], else_=0)).label('sms_fragments_times_multiplier'),
-        func.sum(case([
-            (FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('email_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
-        ], else_=0)).label('letter_totals'),
-        func.sum(case([
-            (FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent * FactBilling.rate)
-        ], else_=0)).label("letter_cost"),
-        func.sum(case(
-            [
-                (FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
-            ], else_=0
-        )).label('letter_sheet_totals')
-    ).filter(
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).group_by(
-        FactBilling.bst_date,
-        FactBilling.service_id,
-        FactBilling.notification_type
-    ).subquery()
-
-    annual_billing = db.session.query(
-        func.max(AnnualBilling.financial_year_start).label('financial_year_start'),
-        AnnualBilling.service_id,
-        AnnualBilling.free_sms_fragment_limit
-    ).filter(
-        AnnualBilling.financial_year_start <= year_end_date
-    ).group_by(
-        AnnualBilling.service_id,
-        AnnualBilling.free_sms_fragment_limit
-    ).subquery()
-
-    results = db.session.query(
-        Service.name.label("service_name"),
-        Service.id.label("service_id"),
-        Service.organisation_id.label("organisation_id"),
-        Organisation.name.label("organisation_name"),
-        annual_billing.c.free_sms_fragment_limit.label("free_allowance"),
-        func.coalesce(func.sum(volume_stats.c.sms_totals), 0).label("sms_notifications"),
-        func.coalesce(func.sum(volume_stats.c.sms_fragments_times_multiplier), 0
-                      ).label("sms_chargeable_units"),
-        func.coalesce(func.sum(volume_stats.c.email_totals), 0).label("email_totals"),
-        func.coalesce(func.sum(volume_stats.c.letter_totals), 0).label("letter_totals"),
-        func.coalesce(func.sum(volume_stats.c.letter_cost), 0).label("letter_cost"),
-        func.coalesce(func.sum(volume_stats.c.letter_sheet_totals), 0).label("letter_sheet_totals")
-    ).select_from(
-        Service
-    ).outerjoin(
-        Organisation, Service.organisation_id == Organisation.id
-    ).join(
-        annual_billing, Service.id == annual_billing.c.service_id
-    ).outerjoin(  # include services without volume
-        volume_stats, Service.id == volume_stats.c.service_id
-    ).filter(
-        Service.restricted.is_(False),
-        Service.count_as_live.is_(True),
-        Service.active.is_(True)
-    ).group_by(
-        Service.id,
-        Service.name,
-        Service.organisation_id,
-        Organisation.name,
-        annual_billing.c.free_sms_fragment_limit
-    ).order_by(
-        Organisation.name,
-        Service.name,
-    ).all()
-
-    return results

app/dao/fact_notification_status_dao.py

@@ -1,89 +1,108 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time
 
-from sqlalchemy import Date, case, func
+from flask import current_app
+from notifications_utils.timezones import convert_bst_to_utc
+from sqlalchemy import case, func, Date
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import extract, literal
+from sqlalchemy.sql.expression import literal, extract
 from sqlalchemy.types import DateTime, Integer
 
 from app import db
-from app.dao.dao_utils import autocommit
 from app.models import (
-    KEY_TYPE_NORMAL,
-    KEY_TYPE_TEAM,
+    FactNotificationStatus,
     KEY_TYPE_TEST,
+    Notification,
     NOTIFICATION_CANCELLED,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_FAILED,
     NOTIFICATION_PENDING,
-    NOTIFICATION_PERMANENT_FAILURE,
     NOTIFICATION_SENDING,
     NOTIFICATION_SENT,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    FactNotificationStatus,
-    Notification,
-    NotificationAllTimeView,
+    NOTIFICATION_PERMANENT_FAILURE,
     Service,
     Template,
 )
+from app.dao.dao_utils import transactional
 from app.utils import (
     get_london_midnight_in_utc,
-    get_london_month_from_utc_column,
     midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
 )
 
 
-@autocommit
-def update_fact_notification_status(process_day, notification_type, service_id):
-    start_date = get_london_midnight_in_utc(process_day)
-    end_date = get_london_midnight_in_utc(process_day + timedelta(days=1))
+def fetch_notification_status_for_day(process_day, notification_type):
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
 
-    # delete any existing rows in case some no longer exist e.g. if all messages are sent
-    FactNotificationStatus.query.filter(
-        FactNotificationStatus.bst_date == process_day,
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.service_id == service_id,
-    ).delete()
+    current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
 
+    all_data_for_process_day = []
+    services = Service.query.all()
+    # for each service query notifications or notification_history for the day, depending on their data retention
+    for service in services:
+        table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
+
+        data_for_service_and_type = query_for_fact_status_data(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+            notification_type=notification_type,
+            service_id=service.id
+        )
+
+        all_data_for_process_day += data_for_service_and_type
+
+    return all_data_for_process_day
+
+
+def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
     query = db.session.query(
-        literal(process_day).label("process_day"),
-        NotificationAllTimeView.template_id,
-        literal(service_id).label("service_id"),
-        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
-        literal(notification_type).label("notification_type"),
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status,
+        table.template_id,
+        table.service_id,
+        func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        table.key_type,
+        table.status,
         func.count().label('notification_count')
     ).filter(
-        NotificationAllTimeView.created_at >= start_date,
-        NotificationAllTimeView.created_at < end_date,
-        NotificationAllTimeView.notification_type == notification_type,
-        NotificationAllTimeView.service_id == service_id,
-        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        table.created_at >= start_date,
+        table.created_at < end_date,
+        table.notification_type == notification_type,
+        table.service_id == service_id,
+        table.key_type != KEY_TYPE_TEST
     ).group_by(
-        NotificationAllTimeView.template_id,
-        NotificationAllTimeView.template_id,
+        table.template_id,
+        table.service_id,
         'job_id',
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status
+        table.key_type,
+        table.status
     )
+    return query.all()
 
-    db.session.connection().execute(
-        insert(FactNotificationStatus.__table__).from_select(
-            [
-                FactNotificationStatus.bst_date,
-                FactNotificationStatus.template_id,
-                FactNotificationStatus.service_id,
-                FactNotificationStatus.job_id,
-                FactNotificationStatus.notification_type,
-                FactNotificationStatus.key_type,
-                FactNotificationStatus.notification_status,
-                FactNotificationStatus.notification_count
-            ],
-            query
+
+@transactional
+def update_fact_notification_status(data, process_day, notification_type):
+    table = FactNotificationStatus.__table__
+    FactNotificationStatus.query.filter(
+        FactNotificationStatus.bst_date == process_day,
+        FactNotificationStatus.notification_type == notification_type
+    ).delete()
+
+    for row in data:
+        stmt = insert(table).values(
+            bst_date=process_day,
+            template_id=row.template_id,
+            service_id=row.service_id,
+            job_id=row.job_id,
+            notification_type=notification_type,
+            key_type=row.key_type,
+            notification_status=row.status,
+            notification_count=row.notification_count,
         )
-    )
+        db.session.connection().execute(stmt)
 
 
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
@@ -411,37 +430,16 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     return query.all()
 
 
-def get_total_notifications_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('emails'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('sms'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('letters'),
+def get_total_sent_notifications_for_day_and_type(day, notification_type):
+    result = db.session.query(
+        func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
+        FactNotificationStatus.notification_type == notification_type,
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
-    ).group_by(
-        FactNotificationStatus.bst_date
-    ).order_by(
-        FactNotificationStatus.bst_date
-    )
-    if start_date and end_date:
-        query = query.filter(
-            FactNotificationStatus.bst_date >= start_date,
-            FactNotificationStatus.bst_date <= end_date
-        )
-    return query.all()
+        FactNotificationStatus.bst_date == day,
+    ).scalar()
+
+    return result or 0
 
 
 def fetch_monthly_notification_statuses_per_service(start_date, end_date):

app/dao/fact_processing_time_dao.py

@@ -1,53 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import case
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import FactProcessingTime
-
-
-@autocommit
-def insert_update_processing_time(processing_time):
-    '''
-    This uses the Postgres upsert to avoid race conditions when two threads try and insert
-    at the same row. The excluded object refers to values that we tried to insert but were
-    rejected.
-    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    '''
-    table = FactProcessingTime.__table__
-    stmt = insert(table).values(
-        bst_date=processing_time.bst_date,
-        messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs
-    )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.bst_date],
-        set_={
-            'messages_total': stmt.excluded.messages_total,
-            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
-            'updated_at': datetime.utcnow()
-        }
-    )
-    db.session.connection().execute(stmt)
-
-
-def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactProcessingTime.bst_date.cast(db.Text).label("date"),
-        FactProcessingTime.messages_total,
-        FactProcessingTime.messages_within_10_secs,
-        case([
-            (
-                FactProcessingTime.messages_total > 0,
-                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
-            ),
-            (FactProcessingTime.messages_total == 0, 100.0)
-        ]).label("percentage")
-    ).filter(
-        FactProcessingTime.bst_date >= start_date,
-        FactProcessingTime.bst_date <= end_date
-    ).order_by(FactProcessingTime.bst_date)
-
-    return query.all()

app/dao/inbound_numbers_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import InboundNumber
 
 
@@ -19,13 +19,13 @@ def dao_get_inbound_number(inbound_number_id):
     return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_to_service(service_id, inbound_number):
     inbound_number.service_id = service_id
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_active_flag(service_id, active):
     inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
     inbound_number.active = active
@@ -33,7 +33,7 @@ def dao_set_inbound_number_active_flag(service_id, active):
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_allocate_number_for_service(service_id, inbound_number_id):
     updated = InboundNumber.query.filter_by(
         id=inbound_number_id,

app/dao/inbound_sms_dao.py

@@ -1,21 +1,15 @@
 from flask import current_app
-from sqlalchemy import and_, desc
-from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import desc, and_
 from sqlalchemy.orm import aliased
+from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    InboundSms,
-    InboundSmsHistory,
-    Service,
-    ServiceDataRetention,
-)
+from app.dao.dao_utils import transactional
+from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
 from app.utils import midnight_n_days_ago
 
 
-@autocommit
+@transactional
 def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
@@ -71,13 +65,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days):
 def _insert_inbound_sms_history(subquery, query_limit=10000):
     offset = 0
     inbound_sms_query = db.session.query(
-        InboundSms.id,
-        InboundSms.created_at,
-        InboundSms.service_id,
-        InboundSms.notify_number,
-        InboundSms.provider_date,
-        InboundSms.provider_reference,
-        InboundSms.provider
+        *[x.name for x in InboundSmsHistory.__table__.c]
     ).filter(InboundSms.id.in_(subquery))
     inbound_sms_count = inbound_sms_query.count()
 
@@ -119,7 +107,7 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
     return deleted
 
 
-@autocommit
+@transactional
 def delete_inbound_sms_older_than_retention():
     current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 

app/dao/invited_org_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedOrganisationUser
 
 

app/dao/invited_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedUser
 
 
@@ -9,7 +9,7 @@ def save_invited_user(invited_user):
     db.session.commit()
 
 
-def get_invited_user_by_service_and_id(service_id, invited_user_id):
+def get_invited_user(service_id, invited_user_id):
     return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()
 
 

app/dao/jobs_dao.py

@@ -2,30 +2,33 @@ import uuid
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.letter_timings import (
-    CANCELLABLE_JOB_LETTER_STATUSES,
-    letter_can_be_cancelled,
+from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
+from sqlalchemy import (
+    asc,
+    desc,
+    func,
+    and_
 )
-from sqlalchemy import and_, asc, desc, func
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.dao.templates_dao import dao_get_template_by_id
+from app.utils import midnight_n_days_ago
+
 from app.models import (
-    JOB_STATUS_CANCELLED,
+    Job,
     JOB_STATUS_FINISHED,
     JOB_STATUS_PENDING,
     JOB_STATUS_SCHEDULED,
     LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    NOTIFICATION_CREATED,
-    FactNotificationStatus,
-    Job,
     Notification,
-    ServiceDataRetention,
     Template,
+    ServiceDataRetention,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_CANCELLED,
+    JOB_STATUS_CANCELLED,
+    FactNotificationStatus
 )
-from app.utils import midnight_n_days_ago
 
 
 def dao_get_notification_outcomes_for_job(service_id, job_id):
@@ -183,7 +186,7 @@ def dao_get_jobs_older_than_data_retention(notification_types):
     return jobs
 
 
-@autocommit
+@transactional
 def dao_cancel_letter_job(job):
     number_of_notifications_cancelled = Notification.query.filter(
         Notification.job_id == job.id

app/dao/letter_branding_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import LetterBranding
 
 
@@ -15,12 +15,12 @@ def dao_get_all_letter_branding():
     return LetterBranding.query.order_by(LetterBranding.name).all()
 
 
-@autocommit
+@transactional
 def dao_create_letter_branding(letter_branding):
     db.session.add(letter_branding)
 
 
-@autocommit
+@transactional
 def dao_update_letter_branding(letter_branding_id, **kwargs):
     letter_branding = LetterBranding.query.get(letter_branding_id)
     for key, value in kwargs.items():

app/dao/notifications_dao.py

@@ -1,53 +1,56 @@
-from datetime import datetime, timedelta
+import functools
 from itertools import groupby
 from operator import attrgetter
+from datetime import (
+    datetime,
+    timedelta,
+)
 
 from botocore.exceptions import ClientError
 from flask import current_app
-from notifications_utils.international_billing_rates import (
-    INTERNATIONAL_BILLING_RATES,
-)
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
     validate_and_format_email_address,
+    InvalidEmailError,
+    try_validate_and_format_phone_number
 )
 from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
-from sqlalchemy import and_, asc, desc, func, or_, union
+from sqlalchemy import (desc, func, asc, and_, or_)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
 from werkzeug.datastructures import MultiDict
 
-from app import create_uuid, db, statsd_client
-from app.dao.dao_utils import autocommit
-from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
+from app import db, create_uuid
+from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
+from app.dao.dao_utils import transactional
+from app.letters.utils import get_letter_pdf_filename
 from app.models import (
-    EMAIL_TYPE,
+    FactNotificationStatus,
+    Notification,
+    NotificationHistory,
+    ProviderDetails,
     KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
     NOTIFICATION_PENDING,
     NOTIFICATION_PENDING_VIRUS_CHECK,
-    NOTIFICATION_PERMANENT_FAILURE,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
+    NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENT,
     SMS_TYPE,
-    FactNotificationStatus,
-    Notification,
-    NotificationHistory,
-    ProviderDetails,
-)
-from app.utils import (
-    escape_special_characters,
-    get_london_midnight_in_utc,
-    midnight_n_days_ago,
+    EMAIL_TYPE,
+    ServiceDataRetention,
+    Service,
 )
+from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters
+from app.clients.sms.firetext import get_message_status_and_reason_from_firetext_code
 
 
 def dao_get_last_date_template_was_used(template_id, service_id):
@@ -72,7 +75,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
     return last_date
 
 
-@autocommit
+@transactional
 def dao_create_notification(notification):
     if not notification.id:
         # need to populate defaulted fields before we create the notification history object
@@ -83,6 +86,25 @@ def dao_create_notification(notification):
     db.session.add(notification)
 
 
+def _decide_permanent_temporary_failure(status, notification, detailed_status_code=None):
+    # If we get failure status from Firetext, we want to know if this is temporary or permanent failure.
+    # So we check the failure code to learn that.
+    # If there is no failure code, or we do not recognise the failure code, we do the following:
+    # if notifitcation goes form status pending to status failure, we mark it as temporary failure;
+    # if notification goes straight to status failure, we mark it as permanent failure.
+    if status == NOTIFICATION_PERMANENT_FAILURE and detailed_status_code not in [None, '000']:
+        try:
+            status, reason = get_message_status_and_reason_from_firetext_code(detailed_status_code)
+            current_app.logger.info(f'Updating notification id {notification.id} to status {status}, reason: {reason}')
+            return status
+        except KeyError:
+            current_app.logger.warning(f'Failure code {detailed_status_code} from Firetext not recognised')
+    # fallback option:
+    if notification.status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+    return status
+
+
 def country_records_delivery(phone_prefix):
     dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
     return dlr and dlr.lower() == 'yes'
@@ -97,7 +119,7 @@ def _update_notification_status(notification, status, detailed_status_code=None)
     return notification
 
 
-@autocommit
+@transactional
 def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
 
@@ -133,7 +155,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
     )
 
 
-@autocommit
+@transactional
 def update_notification_status_by_reference(reference, status):
     # this is used to update letters and emails
     notification = Notification.query.filter(Notification.reference == reference).first()
@@ -155,12 +177,16 @@ def update_notification_status_by_reference(reference, status):
     )
 
 
-@autocommit
+@transactional
 def dao_update_notification(notification):
     notification.updated_at = datetime.utcnow()
     db.session.add(notification)
 
 
+def get_notification_for_job(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -195,6 +221,10 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
     return query.one() if _raise else query.first()
 
 
+def get_notifications(filter_dict=None):
+    return _filter_query(Notification.query, filter_dict=filter_dict)
+
+
 def get_notifications_for_service(
         service_id,
         filter_dict=None,
@@ -208,8 +238,7 @@ def get_notifications_for_service(
         include_from_test_key=False,
         older_than=None,
         client_reference=None,
-        include_one_off=True,
-        error_out=True
+        include_one_off=True
 ):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -248,8 +277,7 @@ def get_notifications_for_service(
     return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
         per_page=page_size,
-        count=count_pages,
-        error_out=error_out,
+        count=count_pages
     )
 
 
@@ -273,42 +301,51 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
-@autocommit
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=50000):
+    current_app.logger.info(
+        'Deleting {} notifications for services with flexible data retention'.format(notification_type))
+
+    flexible_data_retention = ServiceDataRetention.query.filter(
+        ServiceDataRetention.notification_type == notification_type
+    ).all()
+    deleted = 0
+    for f in flexible_data_retention:
+        current_app.logger.info(
+            "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
+
+        day_to_delete_backwards_from = get_london_midnight_in_utc(
+            convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)
+
+        deleted += _move_notifications_to_notification_history(
+            notification_type, f.service_id, day_to_delete_backwards_from, qry_limit)
+
+    current_app.logger.info(
+        'Deleting {} notifications for services without flexible data retention'.format(notification_type))
+
+    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
+    services_with_data_retention = [x.service_id for x in flexible_data_retention]
+    service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
+
+    for service_id in service_ids_to_purge:
+        deleted += _move_notifications_to_notification_history(
+            notification_type, service_id, seven_days_ago, qry_limit)
+
+    current_app.logger.info('Finished deleting {} notifications'.format(notification_type))
+
+    return deleted
+
+
+@transactional
 def insert_notification_history_delete_notifications(
     notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
 ):
-    """
-    Delete up to 50,000 notifications that are past retention for a notification type and service.
-
-
-    Steps are as follows:
-
-    Create a temporary notifications table
-    Populate that table with up to 50k notifications that are to be deleted. (Note: no specified order)
-    Insert everything in the temp table into notification history
-    Delete from notifications if notification id is in the temp table
-    Drop the temp table (automatically when the transaction commits)
-
-    Temporary tables are in a separate postgres schema, and only visible to the current session (db connection,
-    in a celery task there's one connection per thread.)
-    """
     # Setting default query limit to 50,000 which take about 48 seconds on current table size
     # 10, 000 took 11s and 100,000 took 1 min 30 seconds.
-    select_into_temp_table = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, postage, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
+    drop_table_if_exists = """
+        DROP TABLE if exists NOTIFICATION_ARCHIVE
         """
-    select_into_temp_table_for_letters = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
+    select_into_temp_table = """
+         CREATE TEMP TABLE NOTIFICATION_ARCHIVE AS
          SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
              key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
              client_reference, international, phone_prefix, rate_multiplier, notification_status,
@@ -317,7 +354,6 @@ def insert_notification_history_delete_notifications(
         WHERE service_id = :service_id
           AND notification_type = :notification_type
           AND created_at < :timestamp_to_delete_backwards_from
-          AND notification_status NOT IN ('pending-virus-check', 'created', 'sending')
           AND key_type in ('normal', 'team')
         limit :qry_limit
         """
@@ -339,35 +375,31 @@ def insert_notification_history_delete_notifications(
         "qry_limit": qry_limit
     }
 
-    select_to_use = select_into_temp_table_for_letters if notification_type == 'letter' else select_into_temp_table
-    db.session.execute(select_to_use, input_params)
+    db.session.execute(drop_table_if_exists)
+    db.session.execute(select_into_temp_table, input_params)
 
-    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
+    result = db.session.execute("select * from NOTIFICATION_ARCHIVE")
 
     db.session.execute(insert_query)
 
     db.session.execute(delete_query)
 
-    return result
+    db.session.execute("DROP TABLE NOTIFICATION_ARCHIVE")
+    return result.rowcount
 
 
-def move_notifications_to_notification_history(
-    notification_type,
-    service_id,
-    timestamp_to_delete_backwards_from,
-    qry_limit=50000
-):
+def _move_notifications_to_notification_history(notification_type, service_id, day_to_delete_backwards_from, qry_limit):
     deleted = 0
     if notification_type == LETTER_TYPE:
         _delete_letters_from_s3(
-            notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit
+            notification_type, service_id, day_to_delete_backwards_from, qry_limit
         )
     delete_count_per_call = 1
     while delete_count_per_call > 0:
         delete_count_per_call = insert_notification_history_delete_notifications(
             notification_type=notification_type,
             service_id=service_id,
-            timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
+            timestamp_to_delete_backwards_from=day_to_delete_backwards_from,
             qry_limit=qry_limit
         )
         deleted += delete_count_per_call
@@ -376,7 +408,7 @@ def move_notifications_to_notification_history(
     Notification.query.filter(
         Notification.notification_type == notification_type,
         Notification.service_id == service_id,
-        Notification.created_at < timestamp_to_delete_backwards_from,
+        Notification.created_at < day_to_delete_backwards_from,
         Notification.key_type == KEY_TYPE_TEST
     ).delete(synchronize_session=False)
     db.session.commit()
@@ -392,55 +424,77 @@ def _delete_letters_from_s3(
     ).filter(
         Notification.notification_type == notification_type,
         Notification.created_at < date_to_delete_from,
-        Notification.service_id == service_id,
-        # although letters in non completed statuses do have PDFs in s3, they do not exist in the
-        # production-letters-pdf bucket as they never made it that far so we do not try and delete
-        # them from it
-        Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)
+        Notification.service_id == service_id
     ).limit(query_limit).all()
     for letter in letters_to_delete_from_s3:
-        try:
-            letter_pdf = find_letter_pdf_in_s3(letter)
-            letter_pdf.delete()
-        except ClientError:
-            current_app.logger.exception(
-                "Error deleting S3 object for letter: {}".format(letter.id))
-        except LetterPDFNotFound:
-            current_app.logger.warning(
-                "No S3 object to delete for letter: {}".format(letter.id))
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+        # I don't think we need this anymore, we should update the query to get letters sent 7 days ago
+        if letter.sent_at:
+            prefix = get_letter_pdf_filename(reference=letter.reference,
+                                             crown=letter.service.crown,
+                                             created_at=letter.created_at,
+                                             ignore_folder=letter.key_type == KEY_TYPE_TEST,
+                                             postage=letter.postage)
+            s3_objects = get_s3_bucket_objects(bucket_name=bucket_name, subfolder=prefix)
+            for s3_object in s3_objects:
+                try:
+                    remove_s3_object(bucket_name, s3_object['Key'])
+                except ClientError:
+                    current_app.logger.exception(
+                        "Could not delete S3 object with filename: {}".format(s3_object['Key']))
 
 
-@autocommit
+@transactional
 def dao_delete_notifications_by_id(notification_id):
     db.session.query(Notification).filter(
         Notification.id == notification_id
     ).delete(synchronize_session='fetch')
 
 
-def dao_timeout_notifications(cutoff_time, limit=100000):
-    """
-    Set email and SMS notifications (only) to "temporary-failure" status
-    if they're still sending from before the specified cutoff_time.
-    """
-    updated_at = datetime.utcnow()
-    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
-    new_status = NOTIFICATION_TEMPORARY_FAILURE
-
+def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
     notifications = Notification.query.filter(
-        Notification.created_at < cutoff_time,
+        Notification.created_at < timeout_start,
         Notification.status.in_(current_statuses),
-        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
-    ).limit(limit).all()
-
+        Notification.notification_type != LETTER_TYPE
+    ).all()
     Notification.query.filter(
-        Notification.id.in_([n.id for n in notifications]),
+        Notification.created_at < timeout_start,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type != LETTER_TYPE
     ).update(
         {'status': new_status, 'updated_at': updated_at},
         synchronize_session=False
     )
+    return notifications
+
+
+def dao_timeout_notifications(timeout_period_in_seconds):
+    """
+    Timeout SMS and email notifications by the following rules:
+
+    we never sent the notification to the provider for some reason
+        created -> technical-failure
+
+    the notification was sent to the provider but there was not a delivery receipt
+        sending -> temporary-failure
+        pending -> temporary-failure
+
+    Letter notifications are not timed out
+    """
+    timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
+    updated_at = datetime.utcnow()
+    timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
+
+    # Notifications still in created status are marked with a technical-failure:
+    technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
+
+    # Notifications still in sending or pending status are marked with a temporary-failure:
+    temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
+                                              NOTIFICATION_TEMPORARY_FAILURE)
 
     db.session.commit()
-    return notifications
+
+    return technical_failure_notifications, temporary_failure_notifications
 
 
 def is_delivery_slow_for_providers(
@@ -469,7 +523,6 @@ def is_delivery_slow_for_providers(
         ProviderDetails
     ).outerjoin(
         Notification, and_(
-            Notification.notification_type == SMS_TYPE,
             Notification.sent_by == ProviderDetails.identifier,
             Notification.created_at >= created_at,
             Notification.sent_at.isnot(None),
@@ -493,12 +546,15 @@ def is_delivery_slow_for_providers(
         slow_notifications = sum(row.count for row in rows if row.slow)
 
         slow_providers[provider] = (slow_notifications / total_notifications >= threshold)
-        statsd_client.gauge(f'slow-delivery.{provider}.ratio', slow_notifications / total_notifications)
+
+        current_app.logger.info("Slow delivery notifications count for provider {}: {} out of {}. Ratio {}".format(
+            provider, slow_notifications, total_notifications, slow_notifications / total_notifications
+        ))
 
     return slow_providers
 
 
-@autocommit
+@transactional
 def dao_update_notifications_by_reference(references, update_dict):
     updated_count = Notification.query.filter(
         Notification.reference.in_(references)
@@ -526,7 +582,6 @@ def dao_get_notifications_by_recipient_or_reference(
     statuses=None,
     page=1,
     page_size=None,
-    error_out=True,
 ):
 
     if notification_type == SMS_TYPE:
@@ -577,7 +632,7 @@ def dao_get_notifications_by_recipient_or_reference(
     results = db.session.query(Notification)\
         .filter(*filters)\
         .order_by(desc(Notification.created_at))\
-        .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
+        .paginate(page=page, per_page=page_size)
     return results
 
 
@@ -600,15 +655,18 @@ def dao_get_notification_or_history_by_reference(reference):
         ).one()
 
 
-def dao_get_notifications_processing_time_stats(start_date, end_date):
-    """
-    For a given time range, returns the number of notifications sent and the number of
-    those notifications that we processed within 10 seconds
+def dao_get_notifications_by_references(references):
+    return Notification.query.filter(
+        Notification.reference.in_(references)
+    ).all()
 
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+    """
     SELECT
-    count(notifications),
+    count(notification_history),
     coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
     WHERE
     created_at > 'START DATE' AND
     created_at < 'END DATE' AND
@@ -659,20 +717,9 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
+def dao_get_letters_to_be_printed(print_run_deadline, postage):
     """
-    Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
-    to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
-    get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese  start sending off
-    tasks to the notify-ftp app to send them.
-
-    CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
-    there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
-    and we should avoid modifying state of returned objects.
-
-    For more reading:
-    https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
-    https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
+    Return all letters created before the print run deadline that have not yet been sent
     """
     notifications = Notification.query.filter(
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
@@ -680,29 +727,8 @@ def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000
         Notification.status == NOTIFICATION_CREATED,
         Notification.key_type == KEY_TYPE_NORMAL,
         Notification.postage == postage,
-        Notification.billable_units > 0
     ).order_by(
-        Notification.service_id,
         Notification.created_at
-    ).yield_per(query_limit)
-    return notifications
-
-
-def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
-    notifications = db.session.query(
-        func.count(Notification.id).label('letters_count'),
-        func.sum(Notification.billable_units).label('sheets_count'),
-        Notification.postage
-    ).filter(
-        Notification.created_at < convert_bst_to_utc(print_run_deadline),
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL,
-        Notification.billable_units > 0
-    ).group_by(
-        Notification.postage
-    ).order_by(
-        Notification.postage
     ).all()
     return notifications
 
@@ -753,58 +779,14 @@ def dao_precompiled_letters_still_pending_virus_check():
 def _duplicate_update_warning(notification, status):
     current_app.logger.info(
         (
-            'Duplicate callback received for service {service_id}. '
-            'Notification ID {id} with type {type} sent by {sent_by}. '
-            'New status was {new_status}, current status is {old_status}. '
-            'This happened {time_diff} after being first set.'
+            'Duplicate callback received. Notification id {id} received a status update to {new_status}'
+            '{time_diff} after being set to {old_status}. {type} sent by {sent_by}'
         ).format(
             id=notification.id,
             old_status=notification.status,
             new_status=status,
             time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
             type=notification.notification_type,
-            sent_by=notification.sent_by,
-            service_id=notification.service_id
+            sent_by=notification.sent_by
         )
     )
-
-
-def get_service_ids_with_notifications_before(notification_type, timestamp):
-    return {
-        row.service_id
-        for row in db.session.query(
-            Notification.service_id
-        ).filter(
-            Notification.notification_type == notification_type,
-            Notification.created_at < timestamp
-        ).distinct()
-    }
-
-
-def get_service_ids_with_notifications_on_date(notification_type, date):
-    start_date = get_london_midnight_in_utc(date)
-    end_date = get_london_midnight_in_utc(date + timedelta(days=1))
-
-    notification_table_query = db.session.query(
-        Notification.service_id.label('service_id')
-    ).filter(
-        Notification.notification_type == notification_type,
-        # using >= + < is much more efficient than date(created_at)
-        Notification.created_at >= start_date,
-        Notification.created_at < end_date,
-    )
-
-    # Looking at this table is more efficient for historical notifications,
-    # provided the task to populate it has run before they were archived.
-    ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label('service_id')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.bst_date == date,
-    )
-
-    return {
-        row.service_id for row in db.session.query(union(
-            notification_table_query, ft_status_table_query
-        ).subquery()).distinct()
-    }

app/dao/organisation_dao.py

@@ -1,8 +1,14 @@
 from sqlalchemy.sql.expression import func
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.models import Domain, Organisation, Service, User
+from app.dao.dao_utils import VersionOptions, transactional, version_class
+from app.models import (
+    Organisation,
+    Domain,
+    InvitedOrganisationUser,
+    Service,
+    User
+)
 
 
 def dao_get_organisations():
@@ -55,12 +61,12 @@ def dao_get_organisation_by_service_id(service_id):
     return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()
 
 
-@autocommit
+@transactional
 def dao_create_organisation(organisation):
     db.session.add(organisation)
 
 
-@autocommit
+@transactional
 def dao_update_organisation(organisation_id, **kwargs):
 
     domains = kwargs.pop('domains', None)
@@ -105,7 +111,7 @@ def _update_organisation_services(organisation, attribute, only_where_none=True)
         db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_add_service_to_organisation(service, organisation_id):
     organisation = Organisation.query.filter_by(
@@ -119,6 +125,10 @@ def dao_add_service_to_organisation(service, organisation_id):
     db.session.add(service)
 
 
+def dao_get_invited_organisation_user(user_id):
+    return InvitedOrganisationUser.query.filter_by(id=user_id).one()
+
+
 def dao_get_users_for_organisation(organisation_id):
     return db.session.query(
         User
@@ -130,15 +140,10 @@ def dao_get_users_for_organisation(organisation_id):
     ).order_by(User.created_at).all()
 
 
-@autocommit
+@transactional
 def dao_add_user_to_organisation(organisation_id, user_id):
     organisation = dao_get_organisation_by_id(organisation_id)
     user = User.query.filter_by(id=user_id).one()
     user.organisations.append(organisation)
     db.session.add(organisation)
     return user
-
-
-@autocommit
-def dao_remove_user_from_organisation(organisation, user):
-    organisation.users.remove(user)

app/dao/permissions_dao.py

@@ -1,16 +1,16 @@
 from app import db
 from app.dao import DAOClass
 from app.models import (
-    MANAGE_API_KEYS,
-    MANAGE_SETTINGS,
-    MANAGE_TEMPLATES,
+    Permission,
     MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
     SEND_EMAILS,
     SEND_LETTERS,
-    SEND_TEXTS,
-    VIEW_ACTIVITY,
-    Permission,
-)
+    MANAGE_API_KEYS,
+    VIEW_ACTIVITY)
+
 
 # Default permissions for a service
 default_service_permissions = [

app/dao/provider_details_dao.py

@@ -1,18 +1,12 @@
 from datetime import datetime, timedelta
 
-from flask import current_app
 from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import asc, desc, func
+from flask import current_app
 
+from app.dao.dao_utils import transactional
+from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    FactBilling,
-    ProviderDetails,
-    ProviderDetailsHistory,
-    User,
-)
 
 
 def get_provider_details_by_id(provider_details_id):
@@ -36,8 +30,6 @@ def dao_get_provider_versions(provider_id):
         id=provider_id
     ).order_by(
         desc(ProviderDetailsHistory.version)
-    ).limit(
-        100  # limit results instead of adding pagination
     ).all()
 
 
@@ -77,7 +69,7 @@ def _get_sms_providers_for_update(time_threshold):
     return q
 
 
-@autocommit
+@transactional
 def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     """
     Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
@@ -86,8 +78,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     amount_to_reduce_by = 10
     providers_list = _get_sms_providers_for_update(time_threshold)
 
-    if len(providers_list) < 2:
-        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
+    if not providers_list:
         return
 
     providers = {provider.identifier: provider for provider in providers_list}
@@ -104,7 +95,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     _adjust_provider_priority(increased_provider, increased_provider_priority)
 
 
-@autocommit
+@transactional
 def dao_adjust_provider_priority_back_to_resting_points():
     """
     Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
@@ -138,7 +129,7 @@ def get_provider_details_by_notification_type(notification_type, supports_intern
     return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()
 
 
-@autocommit
+@transactional
 def dao_update_provider_details(provider_details):
     _update_provider_details_without_commit(provider_details)
 

app/dao/provider_rates_dao.py

@@ -0,0 +1,11 @@
+from app.models import ProviderRates, ProviderDetails
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
+    db.session.add(provider_rates)

app/dao/returned_letters_dao.py

@@ -1,10 +1,10 @@
 from datetime import datetime
 
-from sqlalchemy import desc, func
+from sqlalchemy import func, desc
 from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import (
     Job,
     Notification,
@@ -28,7 +28,7 @@ def _get_notification_ids_for_references(references):
     return notification_ids + notification_history_ids
 
 
-@autocommit
+@transactional
 def insert_or_update_returned_letters(references):
     data = _get_notification_ids_for_references(references)
     for row in data:

app/dao/service_callback_api_dao.py

@@ -1,15 +1,13 @@
 from datetime import datetime
 
-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    COMPLAINT_CALLBACK_TYPE,
-    DELIVERY_STATUS_CALLBACK_TYPE,
-    ServiceCallbackApi,
-)
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
+from app.models import ServiceCallbackApi
+
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
     service_callback_api.id = create_uuid()
@@ -17,7 +15,7 @@ def save_service_callback_api(service_callback_api):
     db.session.add(service_callback_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -48,6 +46,6 @@ def get_service_complaint_callback_api_for_service(service_id):
     ).first()
 
 
-@autocommit
+@transactional
 def delete_service_callback_api(service_callback_api):
     db.session.delete(service_callback_api)

app/dao/service_data_retention_dao.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceDataRetention
 
 
@@ -28,7 +28,7 @@ def fetch_service_data_retention_by_notification_type(service_id, notification_t
     return data_retention_list
 
 
-@autocommit
+@transactional
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
     new_data_retention = ServiceDataRetention(service_id=service_id,
                                               notification_type=notification_type,
@@ -38,7 +38,7 @@ def insert_service_data_retention(service_id, notification_type, days_of_retenti
     return new_data_retention
 
 
-@autocommit
+@transactional
 def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
     updated_count = ServiceDataRetention.query.filter(
         ServiceDataRetention.id == service_data_retention_id,
@@ -50,7 +50,3 @@ def update_service_data_retention(service_data_retention_id, service_id, days_of
         }
     )
     return updated_count
-
-
-def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
-    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()

app/dao/service_email_reply_to_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceEmailReplyTo
@@ -28,7 +28,7 @@ def dao_get_reply_to_by_id(service_id, reply_to_id):
     return reply_to
 
 
-@autocommit
+@transactional
 def add_reply_to_email_address_for_service(service_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -41,7 +41,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
     return new_reply_to
 
 
-@autocommit
+@transactional
 def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -57,7 +57,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
     return reply_to_update
 
 
-@autocommit
+@transactional
 def archive_reply_to_email_address(service_id, reply_to_id):
     reply_to_archive = ServiceEmailReplyTo.query.filter_by(
         id=reply_to_id,

app/dao/service_inbound_api_dao.py

@@ -1,11 +1,11 @@
 from datetime import datetime
 
-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceInboundApi
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
     service_inbound_api.id = create_uuid()
@@ -13,7 +13,7 @@ def save_service_inbound_api(service_inbound_api):
     db.session.add(service_inbound_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -35,6 +35,6 @@ def get_service_inbound_api_for_service(service_id):
     return ServiceInboundApi.query.filter_by(service_id=service_id).first()
 
 
-@autocommit
+@transactional
 def delete_service_inbound_api(service_inbound_api):
     db.session.delete(service_inbound_api)

app/dao/service_letter_contact_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceLetterContact, Template
 
 
@@ -30,7 +30,7 @@ def dao_get_letter_contact_by_id(service_id, letter_contact_id):
     return letter_contact
 
 
-@autocommit
+@transactional
 def add_letter_contact_for_service(service_id, contact_block, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -45,7 +45,7 @@ def add_letter_contact_for_service(service_id, contact_block, is_default):
     return new_letter_contact
 
 
-@autocommit
+@transactional
 def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
     old_default = _get_existing_default(service_id)
     # if we want to make this the default, ensure there are no other existing defaults
@@ -59,7 +59,7 @@ def update_letter_contact(service_id, letter_contact_id, contact_block, is_defau
     return letter_contact_update
 
 
-@autocommit
+@transactional
 def archive_letter_contact(service_id, letter_contact_id):
     letter_contact_to_archive = ServiceLetterContact.query.filter_by(
         id=letter_contact_id,

app/dao/service_permissions_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServicePermission
 
 
@@ -8,7 +8,7 @@ def dao_fetch_service_permissions(service_id):
         ServicePermission.service_id == service_id).all()
 
 
-@autocommit
+@transactional
 def dao_add_service_permission(service_id, permission):
     service_permission = ServicePermission(service_id=service_id, permission=permission)
     db.session.add(service_permission)

app/dao/service_sms_sender_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceSmsSender
 
@@ -32,7 +32,7 @@ def dao_get_sms_senders_by_service_id(service_id):
     ).order_by(desc(ServiceSmsSender.is_default)).all()
 
 
-@autocommit
+@transactional
 def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
     old_default = _get_existing_default(service_id=service_id)
     if is_default:
@@ -51,7 +51,7 @@ def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_n
     return new_sms_sender
 
 
-@autocommit
+@transactional
 def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -68,7 +68,7 @@ def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default,
     return sms_sender_to_update
 
 
-@autocommit
+@transactional
 def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
     service_sms_sender.sms_sender = sms_sender
     service_sms_sender.inbound_number_id = inbound_number_id
@@ -76,7 +76,7 @@ def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sende
     return service_sms_sender
 
 
-@autocommit
+@transactional
 def archive_sms_sender(service_id, sms_sender_id):
     sms_sender_to_archive = ServiceSmsSender.query.filter_by(
         id=sms_sender_id,

app/dao/service_user_dao.py

@@ -1,6 +1,6 @@
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceUser, User
 
 
@@ -9,13 +9,9 @@ def dao_get_service_user(user_id, service_id):
 
 
 def dao_get_active_service_users(service_id):
-    query = db.session.query(
-        ServiceUser
-    ).join(
-        User, User.id == ServiceUser.user_id
-    ).filter(
-        User.state == 'active',
-        ServiceUser.service_id == service_id
+    query = ServiceUser.query.join(ServiceUser.user).filter(
+        ServiceUser.service_id == service_id,
+        User.state == 'active'
     )
 
     return query.all()
@@ -25,6 +21,6 @@ def dao_get_service_users_by_user_id(user_id):
     return ServiceUser.query.filter_by(user_id=user_id).all()
 
 
-@autocommit
+@transactional
 def dao_update_service_user(service_user):
     db.session.add(service_user)

app/dao/services_dao.py

@@ -1,14 +1,18 @@
 import uuid
 from datetime import date, datetime, timedelta
 
-from flask import current_app
-from sqlalchemy import Float, cast
+from sqlalchemy.sql.expression import asc, case, and_, func
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import and_, asc, case, func
+from sqlalchemy import cast, Float
+from flask import current_app
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
 from app.dao.email_branding_dao import dao_get_email_branding_by_name
 from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
@@ -16,17 +20,6 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    CROWN_ORGANISATION_TYPES,
-    EMAIL_TYPE,
-    INTERNATIONAL_LETTERS,
-    INTERNATIONAL_SMS_TYPE,
-    KEY_TYPE_TEST,
-    LETTER_TYPE,
-    NHS_ORGANISATION_TYPES,
-    NON_CROWN_ORGANISATION_TYPES,
-    NOTIFICATION_PERMANENT_FAILURE,
-    SMS_TYPE,
-    UPLOAD_LETTERS,
     AnnualBilling,
     ApiKey,
     FactBilling,
@@ -38,22 +31,34 @@ from app.models import (
     Organisation,
     Permission,
     Service,
-    ServiceContactList,
-    ServiceEmailReplyTo,
-    ServiceLetterContact,
     ServicePermission,
     ServiceSmsSender,
+    ServiceEmailReplyTo,
+    ServiceContactList,
+    ServiceLetterContact,
     Template,
     TemplateHistory,
     TemplateRedacted,
     User,
     VerifyCode,
+    CROWN_ORGANISATION_TYPES,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NHS_ORGANISATION_TYPES,
+    NON_CROWN_ORGANISATION_TYPES,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    UPLOAD_LETTERS,
+    INTERNATIONAL_LETTERS
 )
 from app.utils import (
     email_address_is_nhs,
     escape_special_characters,
     get_archived_db_column_value,
     get_london_midnight_in_utc,
+    midnight_n_days_ago,
 )
 
 DEFAULT_SERVICE_PERMISSIONS = [
@@ -246,7 +251,7 @@ def dao_fetch_all_services_created_by_user(user_id):
     return query.all()
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -283,7 +288,7 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
     ).one()
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_create_service(
     service,
@@ -317,17 +322,15 @@ def dao_create_service(
     if organisation:
         service.organisation_id = organisation.id
         service.organisation_type = organisation.organisation_type
-
         if organisation.email_branding:
             service.email_branding = organisation.email_branding
 
-        if organisation.letter_branding:
+        if organisation.letter_branding and not service.letter_branding:
             service.letter_branding = organisation.letter_branding
 
     elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
         service.email_branding = dao_get_email_branding_by_name('NHS')
         service.letter_branding = dao_get_letter_branding_by_name('NHS')
-
     if organisation:
         service.crown = organisation.crown
     elif service.organisation_type in CROWN_ORGANISATION_TYPES:
@@ -339,7 +342,7 @@ def dao_create_service(
     db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_update_service(service):
     db.session.add(service)
@@ -421,22 +424,46 @@ def delete_service_and_all_associated_db_objects(service):
     db.session.commit()
 
 
-def dao_fetch_todays_stats_for_service(service_id):
-    today = date.today()
-    start_date = get_london_midnight_in_utc(today)
+def dao_fetch_stats_for_service(service_id, limit_days):
+    # We always want between seven and eight days
+    start_date = midnight_n_days_ago(limit_days)
+    return _stats_for_service_query(service_id).filter(
+        Notification.created_at >= start_date
+    ).all()
 
+
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def fetch_todays_total_message_count(service_id):
+    result = db.session.query(
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        func.date(Notification.created_at) == date.today()
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).first()
+    return 0 if result is None else result.count
+
+
+def _stats_for_service_query(service_id):
     return db.session.query(
         Notification.notification_type,
         Notification.status,
         func.count(Notification.id).label('count')
     ).filter(
         Notification.service_id == service_id,
-        Notification.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= start_date
+        Notification.key_type != KEY_TYPE_TEST
     ).group_by(
         Notification.notification_type,
         Notification.status,
-    ).all()
+    )
 
 
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
@@ -484,7 +511,7 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act
     return query.all()
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -503,7 +530,7 @@ def dao_suspend_service(service_id):
     service.active = False
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_resume_service(service_id):
     service = Service.query.get(service_id)
@@ -531,8 +558,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
         Notification.notification_type == SMS_TYPE,
         func.substr(Notification.normalised_to, 3, 7) == '7700900',
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -540,7 +567,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
     ).all()
 
 
-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
     subquery = db.session.query(
         func.count(Notification.id).label('total_count'),
         Notification.service_id.label('service_id')
@@ -551,8 +578,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.key_type != KEY_TYPE_TEST,
         Notification.notification_type == SMS_TYPE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -577,8 +604,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.notification_type == SMS_TYPE,
         Notification.status == NOTIFICATION_PERMANENT_FAILURE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
         subquery.c.total_count
@@ -587,23 +614,3 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
     )
 
     return query.all()
-
-
-def get_live_services_with_organisation():
-    query = db.session.query(
-        Service.id.label("service_id"),
-        Service.name.label("service_name"),
-        Organisation.id.label("organisation_id"),
-        Organisation.name.label("organisation_name")
-    ).outerjoin(
-        Service.organisation
-    ).filter(
-        Service.count_as_live.is_(True),
-        Service.active.is_(True),
-        Service.restricted.is_(False)
-    ).order_by(
-        Organisation.name,
-        Service.name
-    )
-
-    return query.all()

app/dao/template_folder_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import TemplateFolder
 
 
@@ -14,16 +14,16 @@ def dao_get_valid_template_folders_by_id(folder_ids):
     return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all()
 
 
-@autocommit
+@transactional
 def dao_create_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_update_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_delete_template_folder(template_folder):
     db.session.delete(template_folder)

app/dao/templates_dao.py

@@ -1,22 +1,26 @@
-import uuid
 from datetime import datetime
+import uuid
 
 from flask import current_app
 from sqlalchemy import asc, desc
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.dao.users_dao import get_user_by_id
 from app.models import (
     LETTER_TYPE,
     SECOND_CLASS,
     Template,
     TemplateHistory,
-    TemplateRedacted,
+    TemplateRedacted
 )
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.users_dao import get_user_by_id
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
@@ -38,15 +42,18 @@ def dao_create_template(template):
     db.session.add(template)
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
 def dao_update_template(template):
+    if template.archived:
+        template.folder = None
+
     db.session.add(template)
 
 
-@autocommit
+@transactional
 def dao_update_template_reply_to(template_id, reply_to):
     Template.query.filter_by(id=template_id).update(
         {"service_letter_contact_id": reply_to,
@@ -78,7 +85,7 @@ def dao_update_template_reply_to(template_id, reply_to):
     return template
 
 
-@autocommit
+@transactional
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True
     template.template_redacted.updated_at = datetime.utcnow()

app/dao/uploads_dao.py

@@ -1,18 +1,11 @@
 from datetime import datetime
-
 from flask import current_app
-from sqlalchemy import String, and_, desc, func, literal, text
+from sqlalchemy import and_, desc, func, literal, text, String
 
 from app import db
 from app.models import (
-    JOB_STATUS_CANCELLED,
-    JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    Job,
-    Notification,
-    ServiceDataRetention,
-    Template,
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED, ServiceDataRetention
 )
 from app.utils import midnight_n_days_ago
 
@@ -50,8 +43,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
         func.coalesce(
             Job.processing_started, Job.created_at
-        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
-        Job.contact_list_id.is_(None),
+        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:
         jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
@@ -84,7 +76,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
         Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
+        Template.hidden == True,
         Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
     ]
     if limit_days is not None:

app/dao/users_dao.py

@@ -1,16 +1,16 @@
+from random import (SystemRandom)
+from datetime import (datetime, timedelta)
 import uuid
-from datetime import datetime, timedelta
-from random import SystemRandom
 
 from sqlalchemy import func
 from sqlalchemy.orm import joinedload
 
 from app import db
-from app.dao.dao_utils import autocommit
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_users_by_user_id
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
-from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
+from app.models import (EMAIL_AUTH_TYPE, User, VerifyCode)
 from app.utils import escape_special_characters, get_archived_db_column_value
 
 
@@ -20,23 +20,15 @@ def _remove_values_for_keys_if_present(dict, keys):
 
 
 def create_secret_code():
-    return ''.join(get_non_repeating_random_digits(5))
+    return ''.join(map(str, [SystemRandom().randrange(10) for i in range(5)]))
 
 
-def get_non_repeating_random_digits(length):
-    output = [None] * length
-    for index in range(length):
-        while output[index] in {None, output[index - 1]}:
-            output[index] = str(SystemRandom().randrange(10))
-    return output
-
-
-def save_user_attribute(usr, update_dict=None):
-    db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
+def save_user_attribute(usr, update_dict={}):
+    db.session.query(User).filter_by(id=usr.id).update(update_dict)
     db.session.commit()
 
 
-def save_model_user(user, update_dict=None, password=None, validated_email_access=False):
+def save_model_user(user, update_dict={}, password=None, validated_email_access=False):
     if password:
         user.password = password
         user.password_changed_at = datetime.utcnow()
@@ -44,7 +36,7 @@ def save_model_user(user, update_dict=None, password=None, validated_email_acces
         user.email_access_validated_at = datetime.utcnow()
     if update_dict:
         _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
-        db.session.query(User).filter_by(id=user.id).update(update_dict or {})
+        db.session.query(User).filter_by(id=user.id).update(update_dict)
     else:
         db.session.add(user)
     db.session.commit()
@@ -131,10 +123,12 @@ def reset_failed_login_count(user):
         db.session.commit()
 
 
-def update_user_password(user, password):
+def update_user_password(user, password, validated_email_access=False):
     # reset failed login count - they've just reset their password so should be fine
     user.password = password
     user.password_changed_at = datetime.utcnow()
+    if validated_email_access:
+        user.email_access_validated_at = datetime.utcnow()
     db.session.add(user)
     db.session.commit()
 
@@ -152,7 +146,7 @@ def get_user_and_accounts(user_id):
     ).one()
 
 
-@autocommit
+@transactional
 def dao_archive_user(user):
     if not user_can_be_archived(user):
         msg = "User can’t be removed from a service - check all services have another team member with manage_settings"

app/dao/webauthn_credential_dao.py

@@ -1,40 +0,0 @@
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import WebauthnCredential
-
-
-def dao_get_webauthn_credential_by_user_and_id(user_id, webauthn_credential_id):
-    return WebauthnCredential.query.filter(
-        WebauthnCredential.user_id == user_id,
-        WebauthnCredential.id == webauthn_credential_id
-    ).one()
-
-
-@autocommit
-def dao_create_webauthn_credential(
-    *,
-    user_id,
-    name,
-    credential_data,
-    registration_response,
-):
-    webauthn_credential = WebauthnCredential(
-        user_id=user_id,
-        name=name,
-        credential_data=credential_data,
-        registration_response=registration_response
-    )
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_update_webauthn_credential_name(webauthn_credential, new_name):
-    webauthn_credential.name = new_name
-    db.session.add(webauthn_credential)
-    return webauthn_credential
-
-
-@autocommit
-def dao_delete_webauthn_credential(webauthn_credential):
-    db.session.delete(webauthn_credential)

app/delivery/send_to_providers.py

@@ -1,43 +1,40 @@
 import random
-from datetime import datetime, timedelta
 from urllib import parse
+from datetime import datetime, timedelta
 
-from cachetools import TTLCache, cached
 from flask import current_app
-from notifications_utils.template import (
-    HTMLEmailTemplate,
-    PlainTextEmailTemplate,
-    SMSMessageTemplate,
+from notifications_utils.recipients import (
+    validate_and_format_phone_number,
+    validate_and_format_email_address
 )
+from notifications_utils.template import HTMLEmailTemplate, PlainTextEmailTemplate, SMSMessageTemplate
 
-from app import create_uuid, db, notification_provider_clients, statsd_client
-from app.celery.research_mode_tasks import (
-    send_email_response,
-    send_sms_response,
+from app import clients, statsd_client, create_uuid
+from app.dao.notifications_dao import (
+    dao_update_notification
 )
-from app.dao.email_branding_dao import dao_get_email_branding_by_id
-from app.dao.notifications_dao import dao_update_notification
 from app.dao.provider_details_dao import (
-    dao_reduce_sms_provider_priority,
     get_provider_details_by_notification_type,
+    dao_reduce_sms_provider_priority
 )
+from app.celery.research_mode_tasks import send_sms_response, send_email_response
+from app.dao.templates_dao import dao_get_template_by_id
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
+    SMS_TYPE,
+    KEY_TYPE_TEST,
     BRANDING_BOTH,
     BRANDING_ORG_BANNER,
     EMAIL_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_STATUS_TYPES_COMPLETED,
     NOTIFICATION_TECHNICAL_FAILURE,
-    SMS_TYPE,
+    NOTIFICATION_SENT,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_STATUS_TYPES_COMPLETED
 )
-from app.serialised_models import SerialisedService, SerialisedTemplate
 
 
 def send_sms_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
+    service = notification.service
 
     if not service.active:
         technical_failure(notification=notification)
@@ -46,9 +43,7 @@ def send_sms_to_provider(notification):
     if notification.status == 'created':
         provider = provider_to_use(SMS_TYPE, notification.international)
 
-        template_model = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        )
+        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
 
         template = SMSMessageTemplate(
             template_model.__dict__,
@@ -56,63 +51,41 @@ def send_sms_to_provider(notification):
             prefix=service.name,
             show_prefix=service.prefix_sms,
         )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.get_name(), str(notification.id), notification.to)
 
         else:
             try:
-                # End DB session here so that we don't have a connection stuck open waiting on the call
-                # to one of the SMS providers
-                # We don't want to tie our DB connections being open to the performance of our SMS
-                # providers as a slow down of our providers can cause us to run out of DB connections
-                # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
-                # closing the session (as otherwise it would be reopened immediately)
-                send_sms_kwargs = {
-                    'to': notification.normalised_to,
-                    'content': str(template),
-                    'reference': str(notification.id),
-                    'sender': notification.reply_to_text,
-                    'international': notification.international,
-                }
-                db.session.close()  # no commit needed as no changes to objects have been made above
-                provider.send_sms(**send_sms_kwargs)
+                provider.send_sms(
+                    to=validate_and_format_phone_number(notification.to, international=notification.international),
+                    content=str(template),
+                    reference=str(notification.id),
+                    sender=notification.reply_to_text
+                )
             except Exception as e:
                 notification.billable_units = template.fragment_count
                 dao_update_notification(notification)
-                dao_reduce_sms_provider_priority(provider.name, time_threshold=timedelta(minutes=1))
+                dao_reduce_sms_provider_priority(provider.get_name(), time_threshold=timedelta(minutes=1))
                 raise e
             else:
                 notification.billable_units = template.fragment_count
                 update_notification_to_sending(notification, provider)
 
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
         statsd_client.timing("sms.total-time", delta_seconds)
 
-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("sms.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("sms.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("sms.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("sms.live-key.not-high-volume.total-time", delta_seconds)
-
 
 def send_email_to_provider(notification):
-    service = SerialisedService.from_id(notification.service_id)
-
+    service = notification.service
     if not service.active:
         technical_failure(notification=notification)
         return
     if notification.status == 'created':
         provider = provider_to_use(EMAIL_TYPE)
 
-        template_dict = SerialisedTemplate.from_id_and_service_id(
-            template_id=notification.template_id, service_id=service.id, version=notification.template_version
-        ).__dict__
+        template_dict = dao_get_template_by_id(notification.template_id, notification.template_version).__dict__
 
         html_email = HTMLEmailTemplate(
             template_dict,
@@ -124,8 +97,7 @@ def send_email_to_provider(notification):
             template_dict,
             values=notification.personalisation
         )
-        created_at = notification.created_at
-        key_type = notification.key_type
+
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             notification.reference = str(create_uuid())
             update_notification_to_sending(notification, provider)
@@ -134,42 +106,32 @@ def send_email_to_provider(notification):
             from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
                                                  current_app.config['NOTIFY_EMAIL_DOMAIN'])
 
+            email_reply_to = notification.reply_to_text
+
             reference = provider.send_email(
                 from_address,
-                notification.normalised_to,
+                validate_and_format_email_address(notification.to),
                 plain_text_email.subject,
                 body=str(plain_text_email),
                 html_body=str(html_email),
-                reply_to_address=notification.reply_to_text
+                reply_to_address=validate_and_format_email_address(email_reply_to) if email_reply_to else None,
             )
             notification.reference = reference
             update_notification_to_sending(notification, provider)
-        delta_seconds = (datetime.utcnow() - created_at).total_seconds()
 
-        if key_type == KEY_TYPE_TEST:
-            statsd_client.timing("email.test-key.total-time", delta_seconds)
-        else:
-            statsd_client.timing("email.live-key.total-time", delta_seconds)
-            if service.high_volume:
-                statsd_client.timing("email.live-key.high-volume.total-time", delta_seconds)
-            else:
-                statsd_client.timing("email.live-key.not-high-volume.total-time", delta_seconds)
+        delta_seconds = (datetime.utcnow() - notification.created_at).total_seconds()
+        statsd_client.timing("email.total-time", delta_seconds)
 
 
 def update_notification_to_sending(notification, provider):
     notification.sent_at = datetime.utcnow()
-    notification.sent_by = provider.name
+    notification.sent_by = provider.get_name()
     if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
         notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
     dao_update_notification(notification)
 
 
-provider_cache = TTLCache(maxsize=8, ttl=10)
-
-
-@cached(cache=provider_cache)
-def provider_to_use(notification_type, international=True):
-    international = True # TODO: remove or resolve the functionality of this
+def provider_to_use(notification_type, international=False):
     active_providers = [
         p for p in get_provider_details_by_notification_type(notification_type, international) if p.active
     ]
@@ -180,14 +142,9 @@ def provider_to_use(notification_type, international=True):
         )
         raise Exception("No active {} providers".format(notification_type))
 
-    if len(active_providers) == 1:
-        weights = [100]
-    else:
-        weights = [p.priority for p in active_providers]
+    chosen_provider = random.choices(active_providers, weights=[p.priority for p in active_providers])[0]
 
-    chosen_provider = random.choices(active_providers, weights=weights)[0]
-
-    return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
+    return clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
 
 
 def get_logo_url(base_url, logo_file):
@@ -212,28 +169,25 @@ def get_logo_url(base_url, logo_file):
 
 
 def get_html_email_options(service):
+
     if service.email_branding is None:
         return {
             'govuk_banner': True,
             'brand_banner': False,
         }
-    if isinstance(service, SerialisedService):
-        branding = dao_get_email_branding_by_id(service.email_branding)
-    else:
-        branding = service.email_branding
 
     logo_url = get_logo_url(
         current_app.config['ADMIN_BASE_URL'],
-        branding.logo
-    ) if branding.logo else None
+        service.email_branding.logo
+    ) if service.email_branding.logo else None
 
     return {
-        'govuk_banner': branding.brand_type == BRANDING_BOTH,
-        'brand_banner': branding.brand_type == BRANDING_ORG_BANNER,
-        'brand_colour': branding.colour,
+        'govuk_banner': service.email_branding.brand_type == BRANDING_BOTH,
+        'brand_banner': service.email_branding.brand_type == BRANDING_ORG_BANNER,
+        'brand_colour': service.email_branding.colour,
         'brand_logo': logo_url,
-        'brand_text': branding.text,
-        'brand_name': branding.name,
+        'brand_text': service.email_branding.text,
+        'brand_name': service.email_branding.name,
     }
 
 

app/email_branding/email_branding_schema.py

@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES
 
 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }
 
 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {

app/email_branding/rest.py

@@ -2,16 +2,16 @@ from flask import Blueprint, jsonify, request
 
 from app.dao.email_branding_dao import (
     dao_create_email_branding,
-    dao_get_email_branding_by_id,
     dao_get_email_branding_options,
-    dao_update_email_branding,
-)
-from app.email_branding.email_branding_schema import (
-    post_create_email_branding_schema,
-    post_update_email_branding_schema,
+    dao_get_email_branding_by_id,
+    dao_update_email_branding
 )
 from app.errors import register_errors
 from app.models import EmailBranding
+from app.email_branding.email_branding_schema import (
+    post_create_email_branding_schema,
+    post_update_email_branding_schema
+)
 from app.schema_validation import validate
 
 email_branding_blueprint = Blueprint('email_branding', __name__)

app/errors.py

@@ -1,10 +1,12 @@
-from flask import current_app, json, jsonify
-from jsonschema import ValidationError as JsonSchemaValidationError
-from marshmallow import ValidationError
+from flask import (
+    jsonify,
+    current_app,
+    json)
 from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
-
+from marshmallow import ValidationError
+from jsonschema import ValidationError as JsonSchemaValidationError
 from app.authentication.auth import AuthError
 from app.exceptions import ArchiveValidationError
 

app/events/rest.py

@@ -1,8 +1,13 @@
-from flask import Blueprint, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)
 
-from app.dao.events_dao import dao_create_event
 from app.errors import register_errors
+
 from app.schemas import event_schema
+from app.dao.events_dao import dao_create_event
 
 events = Blueprint('events', __name__, url_prefix='/events')
 register_errors(events)
@@ -11,6 +16,6 @@ register_errors(events)
 @events.route('', methods=['POST'])
 def create_event():
     data = request.get_json()
-    event = event_schema.load(data)
+    event = event_schema.load(data).data
     dao_create_event(event)
-    return jsonify(data=event_schema.dump(event)), 201
+    return jsonify(data=event_schema.dump(event).data), 201

app/govuk_alerts/rest.py

@@ -1,31 +0,0 @@
-from flask import Blueprint, jsonify
-
-from app.dao.broadcast_message_dao import dao_get_all_broadcast_messages
-from app.errors import register_errors
-from app.utils import get_dt_string_or_none
-
-govuk_alerts_blueprint = Blueprint(
-    "govuk-alerts",
-    __name__,
-    url_prefix='/govuk-alerts',
-)
-
-register_errors(govuk_alerts_blueprint)
-
-
-@govuk_alerts_blueprint.route('')
-def get_broadcasts():
-    broadcasts = dao_get_all_broadcast_messages()
-    broadcasts_dict = {"alerts": [{
-        "id": broadcast.id,
-        "reference": broadcast.reference,
-        "channel": broadcast.channel,
-        "content": broadcast.content,
-        "areas": broadcast.areas,
-        "status": broadcast.status,
-        "starts_at": get_dt_string_or_none(broadcast.starts_at),
-        "finishes_at": get_dt_string_or_none(broadcast.finishes_at),
-        "approved_at": get_dt_string_or_none(broadcast.approved_at),
-        "cancelled_at": get_dt_string_or_none(broadcast.cancelled_at),
-    } for broadcast in broadcasts]}
-    return jsonify(broadcasts_dict), 200

app/hashing.py

@@ -1,4 +1,4 @@
-from flask_bcrypt import check_password_hash, generate_password_hash
+from flask_bcrypt import generate_password_hash, check_password_hash
 
 
 def hashpw(password):

app/history_meta.py

@@ -15,11 +15,11 @@ session events.
 
 """
 import datetime
-
-from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, util
 from sqlalchemy.ext.declarative import declared_attr
-from sqlalchemy.orm import attributes, mapper, object_mapper
-from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty
+from sqlalchemy.orm import mapper, attributes, object_mapper
+from sqlalchemy.orm.properties import RelationshipProperty, ColumnProperty
+from sqlalchemy import Table, Column, ForeignKeyConstraint, Integer
+from sqlalchemy import util
 
 
 def col_references_table(col, table):

app/inbound_number/rest.py

@@ -1,10 +1,10 @@
 from flask import Blueprint, jsonify
 
 from app.dao.inbound_numbers_dao import (
-    dao_get_available_inbound_numbers,
-    dao_get_inbound_number_for_service,
     dao_get_inbound_numbers,
-    dao_set_inbound_number_active_flag,
+    dao_get_inbound_number_for_service,
+    dao_get_available_inbound_numbers,
+    dao_set_inbound_number_active_flag
 )
 from app.errors import register_errors
 

app/inbound_sms/inbound_sms_schemas.py

@@ -1,5 +1,5 @@
 get_inbound_sms_for_service_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "schema for parameters allowed when searching for to field=",
     "type": "object",
     "properties": {

app/inbound_sms/rest.py

@@ -1,21 +1,23 @@
-from flask import Blueprint, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)
+
 from notifications_utils.recipients import try_validate_and_format_phone_number
 
 from app.dao.inbound_sms_dao import (
+    dao_get_inbound_sms_for_service,
     dao_count_inbound_sms_for_service,
     dao_get_inbound_sms_by_id,
-    dao_get_inbound_sms_for_service,
-    dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service,
-)
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_by_notification_type,
+    dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service
 )
+from app.dao.service_data_retention_dao import fetch_service_data_retention_by_notification_type
 from app.errors import register_errors
-from app.inbound_sms.inbound_sms_schemas import (
-    get_inbound_sms_for_service_schema,
-)
 from app.schema_validation import validate
 
+from app.inbound_sms.inbound_sms_schemas import get_inbound_sms_for_service_schema
+
 inbound_sms = Blueprint(
     'inbound_sms',
     __name__,

app/invite/rest.py

@@ -0,0 +1,92 @@
+from flask import (
+    Blueprint,
+    request,
+    jsonify,
+    current_app)
+
+from app.config import QueueNames
+from app.dao.invited_user_dao import (
+    save_invited_user,
+    get_invited_user,
+    get_invited_users_for_service
+)
+from app.dao.templates_dao import dao_get_template_by_id
+from app.models import BROADCAST_TYPE, EMAIL_TYPE, KEY_TYPE_NORMAL, Service
+from app.notifications.process_notifications import persist_notification, send_notification_to_queue
+from app.schemas import invited_user_schema
+from app.errors import register_errors
+
+invite = Blueprint('invite', __name__, url_prefix='/service/<service_id>/invite')
+
+register_errors(invite)
+
+
+@invite.route('', methods=['POST'])
+def create_invited_user(service_id):
+    request_json = request.get_json()
+    invited_user, errors = invited_user_schema.load(request_json)
+    save_invited_user(invited_user)
+
+    if invited_user.service.has_permission(BROADCAST_TYPE):
+        template_id = current_app.config['BROADCAST_INVITATION_EMAIL_TEMPLATE_ID']
+    else:
+        template_id = current_app.config['INVITATION_EMAIL_TEMPLATE_ID']
+
+    template = dao_get_template_by_id(template_id)
+    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
+
+    saved_notification = persist_notification(
+        template_id=template.id,
+        template_version=template.version,
+        recipient=invited_user.email_address,
+        service=service,
+        personalisation={
+            'user_name': invited_user.from_user.name,
+            'service_name': invited_user.service.name,
+            'url': invited_user_url(
+                invited_user.id,
+                request_json.get('invite_link_host'),
+            ),
+        },
+        notification_type=EMAIL_TYPE,
+        api_key_id=None,
+        key_type=KEY_TYPE_NORMAL,
+        reply_to_text=invited_user.from_user.email_address
+    )
+
+    send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+
+    return jsonify(data=invited_user_schema.dump(invited_user).data), 201
+
+
+@invite.route('', methods=['GET'])
+def get_invited_users_by_service(service_id):
+    invited_users = get_invited_users_for_service(service_id)
+    return jsonify(data=invited_user_schema.dump(invited_users, many=True).data), 200
+
+
+@invite.route('/<invited_user_id>', methods=['GET'])
+def get_invited_user_by_service(service_id, invited_user_id):
+    invited_user = get_invited_user(service_id, invited_user_id)
+    return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+
+
+@invite.route('/<invited_user_id>', methods=['POST'])
+def update_invited_user(service_id, invited_user_id):
+    fetched = get_invited_user(service_id=service_id, invited_user_id=invited_user_id)
+
+    current_data = dict(invited_user_schema.dump(fetched).data.items())
+    current_data.update(request.get_json())
+    update_dict = invited_user_schema.load(current_data).data
+    save_invited_user(update_dict)
+    return jsonify(data=invited_user_schema.dump(fetched).data), 200
+
+
+def invited_user_url(invited_user_id, invite_link_host=None):
+    from notifications_utils.url_safe_token import generate_token
+    token = generate_token(str(invited_user_id), current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT'])
+
+    if invite_link_host is None:
+        invite_link_host = current_app.config['ADMIN_BASE_URL']
+
+    return '{0}/invitation/{1}'.format(invite_link_host, token)

app/job/rest.py

@@ -1,44 +1,43 @@
 import dateutil
 import pytz
-from flask import Blueprint, current_app, jsonify, request
+from flask import (
+    Blueprint,
+    jsonify,
+    request,
+    current_app
+)
 
 from app.aws.s3 import get_job_metadata_from_s3
-from app.celery.tasks import process_job
-from app.config import QueueNames
-from app.dao.fact_notification_status_dao import (
-    fetch_notification_statuses_for_job,
-)
 from app.dao.jobs_dao import (
-    can_letter_job_be_cancelled,
-    dao_cancel_letter_job,
     dao_create_job,
-    dao_get_future_scheduled_job_by_id_and_service_id,
+    dao_update_job,
     dao_get_job_by_service_id_and_job_id,
     dao_get_jobs_by_service_id,
+    dao_get_future_scheduled_job_by_id_and_service_id,
     dao_get_notification_outcomes_for_job,
     dao_get_scheduled_job_stats,
-    dao_update_job,
-)
-from app.dao.notifications_dao import (
-    dao_get_notification_count_for_job_id,
-    get_notifications_for_job,
+    dao_cancel_letter_job,
+    can_letter_job_be_cancelled
 )
+from app.dao.fact_notification_status_dao import fetch_notification_statuses_for_job
 from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
-from app.errors import InvalidRequest, register_errors
-from app.models import (
-    JOB_STATUS_CANCELLED,
-    JOB_STATUS_PENDING,
-    JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
-)
+from app.dao.notifications_dao import dao_get_notification_count_for_job_id, get_notifications_for_job
 from app.schemas import (
     job_schema,
-    notification_with_template_schema,
-    notifications_filter_schema,
     unarchived_template_schema,
+    notifications_filter_schema,
+    notification_with_template_schema
 )
-from app.utils import midnight_n_days_ago, pagination_links
+from app.celery.tasks import process_job
+from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANCELLED, LETTER_TYPE
+from app.utils import pagination_links, midnight_n_days_ago
+from app.config import QueueNames
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
 
 job_blueprint = Blueprint('job', __name__, url_prefix='/service/<uuid:service_id>/job')
 
@@ -50,7 +49,7 @@ register_errors(job_blueprint)
 def get_job_by_service_and_job_id(service_id, job_id):
     job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
     statistics = dao_get_notification_outcomes_for_job(service_id, job_id)
-    data = job_schema.dump(job)
+    data = job_schema.dump(job).data
 
     data['statistics'] = [{'status': statistic[1], 'count': statistic[0]} for statistic in statistics]
 
@@ -79,7 +78,7 @@ def cancel_letter_job(service_id, job_id):
 
 @job_blueprint.route('/<job_id>/notifications', methods=['GET'])
 def get_all_notifications_for_service_job(service_id, job_id):
-    data = notifications_filter_schema.load(request.args)
+    data = notifications_filter_schema.load(request.args).data
     page = data['page'] if 'page' in data else 1
     page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
     paginated_notifications = get_notifications_for_job(
@@ -97,7 +96,7 @@ def get_all_notifications_for_service_job(service_id, job_id):
     if data.get('format_for_csv'):
         notifications = [notification.serialize_for_csv() for notification in paginated_notifications.items]
     else:
-        notifications = notification_with_template_schema.dump(paginated_notifications.items, many=True)
+        notifications = notification_with_template_schema.dump(paginated_notifications.items, many=True).data
 
     return jsonify(
         notifications=notifications,
@@ -173,7 +172,7 @@ def create_job(service_id):
 
     data.update({"template_version": template.version})
 
-    job = job_schema.load(data)
+    job = job_schema.load(data).data
 
     if job.scheduled_for:
         job.job_status = JOB_STATUS_SCHEDULED
@@ -185,7 +184,7 @@ def create_job(service_id):
     if job.job_status == JOB_STATUS_PENDING:
         process_job.apply_async([str(job.id)], {'sender_id': sender_id}, queue=QueueNames.JOBS)
 
-    job_json = job_schema.dump(job)
+    job_json = job_schema.dump(job).data
     job_json['statistics'] = []
 
     return jsonify(data=job_json), 201
@@ -219,7 +218,7 @@ def get_paginated_jobs(
         statuses=statuses,
         contact_list_id=contact_list_id,
     )
-    data = job_schema.dump(pagination.items, many=True)
+    data = job_schema.dump(pagination.items, many=True).data
     for job_data in data:
         start = job_data['processing_started']
         start = dateutil.parser.parse(start).replace(tzinfo=None) if start else None

app/letter_branding/letter_branding_rest.py

@@ -1,17 +1,15 @@
 from celery import current_app
-from flask import Blueprint, jsonify, request
 from sqlalchemy.exc import IntegrityError
 
+from flask import Blueprint, jsonify, request
+
 from app.dao.letter_branding_dao import (
-    dao_create_letter_branding,
-    dao_get_all_letter_branding,
-    dao_get_letter_branding_by_id,
+    dao_get_all_letter_branding, dao_create_letter_branding,
     dao_update_letter_branding,
+    dao_get_letter_branding_by_id
 )
 from app.errors import register_errors
-from app.letter_branding.letter_branding_schema import (
-    post_letter_branding_schema,
-)
+from app.letter_branding.letter_branding_schema import post_letter_branding_schema
 from app.models import LetterBranding
 from app.schema_validation import validate