remove unused pytest flags

.cfignore

@@ -1,109 +0,0 @@
-# from deploy-exclude.lst
-
-*__pycache__*
-.git/*
-app/assets/*
-bower_components/*
-cache/*
-.cache/*
-node_modules/*
-target/*
-venv/*
-build/*
-.envrc
-tests/.cache/*
-.cf/*
-
-# from .gitignore
-
-queues.csv
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-
-.venv/
-venv/
-venv-freeze/
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-/cache
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-.pytest_cache
-coverage.xml
-test_results.xml
-*,cover
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-.idea/
-.vscode
-
-# Mac
-*.DS_Store
-environment.sh
-.envrc
-.env
-.env*
-varsfile
-
-celerybeat-schedule
-
-# CloudFoundry
-.cf
-varsfile*
-.secret*
-
-/scripts/run_my_tests.sh
-
-# Terraform
-.terraform.lock.hcl
-**/.terraform/*
-secrets.auto.tfvars
-terraform.tfstate
-terraform.tfstate.backup

.cfignore

@@ -0,0 +1 @@
+.gitignore

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html
+# W503: line break before binary operator
+exclude = venv*,__pycache__,node_modules,cache,migrations,build
+ignore = W503
+max-complexity = 14
+max-line-length = 120

.github/actions/deploy-proxy/action.yml

@@ -1,42 +0,0 @@
-name: Deploy egress proxy
-description: Set egress space security groups and deploy proxy
-inputs:
-  cf_space:
-    description: The space the target app exists in.
-    required: true
-  app:
-    description: application name to be proxied.
-    required: true
-  proxy_repo:
-    description: git repo for cg-egress-proxy
-    default: https://github.com/GSA-TTS/cg-egress-proxy.git
-  proxy_version:
-    description: git ref to be deployed
-    default: main
-runs:
-  using: composite
-  steps:
-    - name: Set restricted space egress
-      shell: bash
-      run: ./terraform/set_space_egress.sh -t -s ${{ inputs.cf_space }}
-    - name: Set public space egress
-      shell: bash
-      run: ./terraform/set_space_egress.sh -p -s ${{ inputs.cf_space }}-egress
-    - name: Create temp directory
-      shell: bash
-      id: create-temp-dir
-      run: echo "path=$(mktemp -d -t egress-XXXXXXXXXX --tmpdir=$RUNNER_TEMP)" >> $GITHUB_OUTPUT
-    - name: Clone cg-egress-proxy
-      shell: bash
-      run: git clone ${{ inputs.proxy_repo }} ${{ steps.create-temp-dir.outputs.path }}
-    - name: Switch to deploy ref
-      shell: bash
-      working-directory: ${{ steps.create-temp-dir.outputs.path }}
-      run: git checkout ${{ inputs.proxy_version }}
-    - name: Copy config files
-      shell: bash
-      run: cp ./deploy-config/egress_proxy/${{ inputs.app }}.*.acl ${{ steps.create-temp-dir.outputs.path }}
-    - name: Build and deploy proxy
-      shell: bash
-      working-directory: ${{ steps.create-temp-dir.outputs.path }}
-      run: make && ./bin/cf-deployproxy -a ${{ inputs.app }} -p egress-proxy -e egress_proxy

.github/actions/setup-project/action.yml

@@ -1,18 +0,0 @@
-name: Set up project
-description: Setup python & install dependencies
-runs:
-  using: composite
-  steps:
-    - name: Install container dependencies
-      shell: bash
-      run: |
-        sudo apt-get update \
-        && sudo apt-get install -y --no-install-recommends \
-        libcurl4-openssl-dev
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-    - name: Install pipenv
-      shell: bash
-      run: pip install --upgrade pipenv

.github/workflows/checks.yml

@@ -1,126 +0,0 @@
-name: Run checks
-
-on: [push]
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  NOTIFY_ENVIRONMENT: test
-  NEW_RELIC_CONFIG_FILE: newrelic.ini
-  NEW_RELIC_ENVIRONMENT: test
-  FLASK_APP: application.py
-  WERKZEUG_DEBUG_PIN: off
-  REDIS_ENABLED: 0
-  AWS_US_TOLL_FREE_NUMBER: "+18556438890"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: ./.github/actions/setup-project
-    - name: Install application dependencies
-      run: make bootstrap
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-    - name: Run style checks
-      run: pipenv run flake8 .
-    - name: Check imports alphabetized
-      run: pipenv run isort --check-only ./app ./tests
-    - name: Run tests
-      run: pipenv run pytest -n4 --maxfail=10
-      env:
-        SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-
-  validate-new-relic-config:
-    runs-on: ubuntu-latest
-    environment: staging
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install pipenv packages
-        run: pipenv install --dev
-      - name: Validate NewRelic config
-        env:
-          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-          # Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
-          NEW_RELIC_ENVIRONMENT: staging
-        run: pipenv run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
-
-  pip-audit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Create requirements.txt
-        run: pipenv requirements > requirements.txt
-      - uses: pypa/gh-action-pip-audit@v1.0.4
-        with:
-          inputs: requirements.txt
-
-  static-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install bandit
-        run: pip install bandit
-      - name: Run scan
-        run: bandit -r app/ --confidence-level medium
-
-  dynamic-scan:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install application dependencies
-        run: make bootstrap
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run server
-        run: make run-flask &
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP Baseline Scan
-        uses: zaproxy/action-api-scan@v0.1.1
-        with:
-          docker_name: 'owasp/zap2docker-stable'
-          target: 'http://localhost:6011/docs/openapi.yml'
-          fail_action: true
-          allow_issue_writing: false
-          rules_file_name: 'zap.conf'
-          cmd_options: '-I'

.github/workflows/daily_checks.yml

@@ -1,80 +0,0 @@
-name: Run daily scans
-
-on:
-  schedule:
-    # cron format: 'minute hour dayofmonth month dayofweek'
-    # this will run at noon UTC every day (7am EST / 8am EDT)
-    - cron: '0 12 * * *'
-
-permissions:
-  contents: read
-
-env:
-  DEBUG: True
-  NOTIFY_ENVIRONMENT: test
-  NEW_RELIC_CONFIG_FILE: newrelic.ini
-  NEW_RELIC_ENVIRONMENT: test
-  FLASK_APP: application.py
-  WERKZEUG_DEBUG_PIN: off
-  REDIS_ENABLED: 0
-  AWS_US_TOLL_FREE_NUMBER: "+18556438890"
-
-jobs:
-  pip-audit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Create requirements.txt
-        run: pipenv requirements > requirements.txt
-      - uses: pypa/gh-action-pip-audit@v1.0.4
-        with:
-          inputs: requirements.txt
-
-  static-scan:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install bandit
-        run: pip install bandit
-      - name: Run scan
-        run: bandit -r app/ --confidence-level medium
-
-  dynamic-scan:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: user
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: test_notification_api
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup-project
-      - name: Install application dependencies
-        run: make bootstrap
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run server
-        run: make run-flask &
-        env:
-          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
-      - name: Run OWASP Baseline Scan
-        uses: zaproxy/action-api-scan@v0.1.1
-        with:
-          docker_name: 'owasp/zap2docker-weekly'
-          target: 'http://localhost:6011/docs/openapi.yml'
-          fail_action: true
-          allow_issue_writing: false
-          rules_file_name: 'zap.conf'
-          cmd_options: '-I'

.github/workflows/deploy-demo.yml

@@ -1,83 +0,0 @@
-name: Deploy to demo environment
-
-on:
-  push:
-    branches: [ production ]
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    environment: demo
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-
-    - name: Check for changes to Terraform
-      id: changed-terraform-files
-      uses: tj-actions/changed-files@v34
-      with:
-        files: |
-          terraform/demo
-          terraform/shared
-          .github/workflows/deploy-demo.yml
-    - name: Terraform init
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/demo
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-      run: terraform init
-    - name: Terraform apply
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/demo
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-        TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-      run: terraform apply -auto-approve -input=false
-
-    - uses: ./.github/actions/setup-project
-    - name: Install application dependencies
-      run: make bootstrap
-
-    - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing
-      run: pipenv requirements > requirements.txt
-
-    - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
-      env:
-        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
-        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-      with:
-        cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-        cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        cf_org: gsa-tts-benefits-studio-prototyping
-        cf_space: notify-demo
-        push_arguments: >-
-          --vars-file deploy-config/demo.yml
-          --var DANGEROUS_SALT="$DANGEROUS_SALT"
-          --var SECRET_KEY="$SECRET_KEY"
-          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-
-    - name: Check for changes to egress config
-      id: changed-egress-config
-      uses: tj-actions/changed-files@v34
-      with:
-        files: |
-          deploy-config/egress_proxy/notify-api-demo.*.acl
-          .github/actions/deploy-proxy/action.yml
-          .github/workflows/deploy-demo.yml
-    - name: Deploy egress proxy
-      if: steps.changed-egress-config.outputs.any_changed == 'true'
-      uses: ./.github/actions/deploy-proxy
-      with:
-        cf_space: notify-demo
-        app: notify-api-demo

.github/workflows/deploy.yml

@@ -1,96 +0,0 @@
-name: Deploy to staging environment
-
-on:
-  workflow_run:
-    workflows: [ Run checks ]
-    types:
-      - completed
-    branches: [ main ] # Redundant, workflow_run events are only triggered on default branch (`main`)
-
-permissions:
-  contents: read
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-
-    environment: staging
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-
-    - name: Check for changes to Terraform
-      id: changed-terraform-files
-      uses: tj-actions/changed-files@v34
-      with:
-        files: |
-          terraform/staging
-          terraform/shared
-          .github/workflows/deploy.yml
-    - name: Terraform init
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/staging
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-      run: terraform init
-    - name: Terraform apply
-      if: steps.changed-terraform-files.outputs.any_changed == 'true'
-      working-directory: terraform/staging
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-        TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-      run: terraform apply -auto-approve -input=false
-
-    - uses: ./.github/actions/setup-project
-    - name: Install application dependencies
-      run: make bootstrap
-
-    - name: Create requirements.txt because Cloud Foundry does a weird pipenv thing
-      run: pipenv requirements > requirements.txt
-
-    - name: Deploy to cloud.gov
-      uses: 18f/cg-deploy-action@main
-      env:
-        DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
-        SECRET_KEY: ${{ secrets.SECRET_KEY }}
-        ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
-        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-      with:
-        cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-        cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        cf_org: gsa-tts-benefits-studio-prototyping
-        cf_space: notify-staging
-        push_arguments: >-
-          --vars-file deploy-config/staging.yml
-          --var DANGEROUS_SALT="$DANGEROUS_SALT"
-          --var SECRET_KEY="$SECRET_KEY"
-          --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
-          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
-
-    - name: Check for changes to egress config
-      id: changed-egress-config
-      uses: tj-actions/changed-files@v34
-      with:
-        files: |
-          deploy-config/egress_proxy/notify-api-staging.*.acl
-          .github/actions/deploy-proxy/action.yml
-          .github/workflows/deploy.yml
-    - name: Deploy egress proxy
-      if: steps.changed-egress-config.outputs.any_changed == 'true'
-      uses: ./.github/actions/deploy-proxy
-      with:
-        cf_space: notify-staging
-        app: notify-api-staging
-
-  bail:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - uses: actions/github-script@v6
-        with:
-          script: core.setFailed('Checks failed, not deploying')

.github/workflows/drift.yml

@@ -1,66 +0,0 @@
-name: Verify Infrastructure
-
-on:
-  schedule:
-    # cron format: 'minute hour dayofmonth month dayofweek'
-    # this will run at noon UTC every day (7am EST / 8am EDT)
-    - cron: '0 12 * * *'
-
-jobs:
-  check_staging_drift:
-    runs-on: ubuntu-latest
-    name: Check for drift of staging terraform configuration
-    environment: staging
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Check for drift
-        uses: dflook/terraform-check@v1
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        with:
-          path: terraform/staging
-
-  check_demo_drift:
-    runs-on: ubuntu-latest
-    name: Check for drift of demo terraform configuration
-    environment: demo
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: 'production'
-
-      - name: Check for drift
-        uses: dflook/terraform-check@v1
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        with:
-          path: terraform/demo
-
-  # check_prod_drift:
-  #   runs-on: ubuntu-latest
-  #   name: Check for drift of production terraform configuration
-  #   environment: production
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v3
-  #       with:
-  #         ref: 'production'
-
-  #     - name: Check for drift
-  #       uses: dflook/terraform-check@v1
-  #       env:
-  #         AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-  #         AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-  #         TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-  #         TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-  #       with:
-  #         path: terraform/production

.github/workflows/restage-apps.yml

@@ -1,36 +0,0 @@
----
-name: Restage apps
-
-on:
-  workflow_dispatch:
-    inputs:
-      environment:
-        description: "Which environment needs to be restaged"
-        required: true
-        default: staging
-        type: environment
-
-jobs:
-  restage_apps:
-    runs-on: ubuntu-latest
-    environment: ${{ inputs.environment }}
-    strategy:
-      matrix:
-        app: ["api", "admin"]
-    steps:
-      - name: Restage ${{matrix.app}}
-        uses: 18f/cg-deploy-action@main
-        with:
-          cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-          cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio-prototyping
-          cf_space: notify-${{ inputs.environment }}
-          full_command: "cf restage --strategy rolling notify-${{matrix.app}}-${{inputs.environment}}"
-      - name: Restage ${{matrix.app}} egress
-        uses: 18f/cg-deploy-action@main
-        with:
-          cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
-          cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-          cf_org: gsa-tts-benefits-studio-prototyping
-          cf_space: notify-${{ inputs.environment }}-egress
-          full_command: "cf restage --strategy rolling egress-proxy-notify-${{matrix.app}}-${{inputs.environment}}"

.github/workflows/terraform-demo.yml

@@ -1,79 +0,0 @@
-name: Run Terraform plan in demo
-
-on:
-  pull_request:
-    branches: [ production ]
-    paths: [ 'terraform/**' ]
-
-defaults:
-  run:
-    working-directory: terraform/demo
-
-jobs:
-  terraform:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    environment: demo
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Terraform format
-        id: format
-        run: terraform fmt -check
-
-      - name: Terraform init
-        id: init
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-
-      - name: Terraform validate
-        id: validation
-        run: terraform validate -no-color
-
-      - name: Terraform plan
-        id: plan
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
-
-      - name: Read Terraform plan output file
-        id: terraform_output
-        uses: juliangruber/read-file-action@v1
-        if: ${{ always() }}
-        with:
-          path: ./terraform/demo/plan_output.txt
-
-      # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
-      - name: Update PR
-        uses: actions/github-script@v6
-        # we would like to update the PR even when a prior step failed
-        if: ${{ always() }}
-        with:
-          script: |
-            const output = `Terraform Format and Style: ${{ steps.format.outcome }}
-            Terraform Initialization: ${{ steps.init.outcome }}
-            Terraform Validation: ${{ steps.validation.outcome }}
-            Terraform Plan: ${{ steps.plan.outcome }}
-
-            <details><summary>Show Plan</summary>
-
-            \`\`\`\n
-            ${{ steps.terraform_output.outputs.content }}
-            \`\`\`
-
-            </details>
-
-            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: output
-            })

.github/workflows/terraform-production.yml

@@ -1,79 +0,0 @@
-name: Run Terraform plan in production
-
-on:
-  pull_request:
-    branches: [ production-disabled-for-now ]
-    paths: [ 'terraform/**' ]
-
-defaults:
-  run:
-    working-directory: terraform/production
-
-jobs:
-  terraform:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    environment: production
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Terraform format
-        id: format
-        run: terraform fmt -check
-
-      - name: Terraform init
-        id: init
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-
-      - name: Terraform validate
-        id: validation
-        run: terraform validate -no-color
-
-      - name: Terraform plan
-        id: plan
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
-
-      - name: Read Terraform plan output file
-        id: terraform_output
-        uses: juliangruber/read-file-action@v1
-        if: ${{ always() }}
-        with:
-          path: ./terraform/production/plan_output.txt
-
-      # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
-      - name: Update PR
-        uses: actions/github-script@v6
-        # we would like to update the PR even when a prior step failed
-        if: ${{ always() }}
-        with:
-          script: |
-            const output = `Terraform Format and Style: ${{ steps.format.outcome }}
-            Terraform Initialization: ${{ steps.init.outcome }}
-            Terraform Validation: ${{ steps.validation.outcome }}
-            Terraform Plan: ${{ steps.plan.outcome }}
-
-            <details><summary>Show Plan</summary>
-
-            \`\`\`\n
-            ${{ steps.terraform_output.outputs.content }}
-            \`\`\`
-
-            </details>
-
-            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: output
-            })

.github/workflows/terraform-staging.yml

@@ -1,79 +0,0 @@
-name: Run Terraform plan in staging
-
-on:
-  pull_request:
-    branches: [ main ]
-    paths: [ 'terraform/**' ]
-
-defaults:
-  run:
-    working-directory: terraform/staging
-
-jobs:
-  terraform:
-    name: Terraform plan
-    runs-on: ubuntu-latest
-    environment: staging
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Terraform format
-        id: format
-        run: terraform fmt -check
-
-      - name: Terraform init
-        id: init
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-        run: terraform init
-
-      - name: Terraform validate
-        id: validation
-        run: terraform validate -no-color
-
-      - name: Terraform plan
-        id: plan
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }}
-          TF_VAR_cf_user: ${{ secrets.CLOUDGOV_USERNAME }}
-          TF_VAR_cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
-        run: terraform plan -no-color -input=false 2>&1 | tee plan_output.txt
-
-      - name: Read Terraform plan output file
-        id: terraform_output
-        uses: juliangruber/read-file-action@v1
-        if: ${{ always() }}
-        with:
-          path: ./terraform/staging/plan_output.txt
-
-      # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow
-      - name: Update PR
-        uses: actions/github-script@v6
-        # we would like to update the PR even when a prior step failed
-        if: ${{ always() }}
-        with:
-          script: |
-            const output = `Terraform Format and Style: ${{ steps.format.outcome }}
-            Terraform Initialization: ${{ steps.init.outcome }}
-            Terraform Validation: ${{ steps.validation.outcome }}
-            Terraform Plan: ${{ steps.plan.outcome }}
-
-            <details><summary>Show Plan</summary>
-
-            \`\`\`\n
-            ${{ steps.terraform_output.outputs.content }}
-            \`\`\`
-
-            </details>
-
-            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
-
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: output
-            })

.gitignore

@@ -4,7 +4,6 @@ queues.csv
 __pycache__/
 *.py[cod]
 
-.venv/
 venv/
 venv-freeze/
 
@@ -57,7 +56,6 @@ test_results.xml
 
 # Django stuff:
 *.log
-/logs/*
 
 # Sphinx documentation
 docs/_build/
@@ -71,22 +69,10 @@ target/
 *.DS_Store
 environment.sh
 .envrc
-.env
-.env*
-varsfile
 
 celerybeat-schedule
 
 # CloudFoundry
 .cf
-varsfile*
-.secret*
 
 /scripts/run_my_tests.sh
-
-# Terraform
-.terraform.lock.hcl
-**/.terraform/*
-secrets.auto.tfvars
-terraform.tfstate
-terraform.tfstate.backup

.profile

@@ -1,7 +0,0 @@
-##
-# Cloud Foundry app initialization script
-# https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile
-##
-
-export https_proxy=$egress_proxy
-export NEW_RELIC_PROXY_HOST=$egress_proxy

.python-version

@@ -1 +0,0 @@
-3.9

.pyup.yml

@@ -0,0 +1,8 @@
+# see https://pyup.io/docs/configuration/ for all available options
+
+schedule: "every week on wednesday"
+
+search: False
+requirements:
+  - requirements-app.txt
+  - requirements-dev.txt

CONTRIBUTING.md

@@ -1,19 +0,0 @@
-# Welcome!
-
-We're so glad you're thinking about contributing to a [open source project of the U.S. government](https://code.gov/)! If you're unsure about anything, just ask -- or submit the issue or pull request anyway. The worst that can happen is you'll be politely asked to change something. We love all friendly contributions.
-
-We encourage you to read this project's CONTRIBUTING policy (you are here), its [LICENSE](LICENSE.md), and its [README](README.md).
-
-## Policies
-
-We want to ensure a welcoming environment for all of our projects. Our staff follow the [TTS Code of Conduct](https://18f.gsa.gov/code-of-conduct/) and all contributors should do the same.
-
-This project is adhering to the [18F Open Source Policy](https://github.com/18f/open-source-policy).
-
-As part of a U.S. government agency, the General Services Administration (GSA)’s Technology Transformation Services (TTS) takes seriously our responsibility to protect the public’s information, including financial and personal information, from unwarranted disclosure. For more information about security and vulnerability disclosure for our projects, please read our [18F Vulnerability Disclosure Policy](https://18f.gsa.gov/vulnerability-disclosure-policy/).
-
-## Public domain
-
-This project is in the public domain within the United States, and applicable copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
-
-All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest.

LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Government Digital Service
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

LICENSE.md

@@ -1,53 +0,0 @@
-# License
-
-## A few parts of this project are not in the public domain
-
-### Copyright ownership of Works created by U.S. federal employees
-
-This repository contains both the original work for [GOV.UK Notify](https://github.com/alphagov/notifications-api) (the Software, as defined in the MIT License) and modifications made by the General Services Administration (GSA). This repository combines the Software and these GSA modifications, which is a “joint work” of the Software and modifications combined into a single work.
-
-As a work of the United States Government, the software modifications made by GSA are not subject to copyright within the United States. Additionally, GSA waives copyright and related rights in its software modifications worldwide through the [CC0 1.0 Universal Public Domain Dedication](https://creativecommons.org/publicdomain/zero/1.0/).
-
-The Software remains subject to copyright under the MIT License. This consists of all work until [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) on June 1, 2022.
-
-#### Full license text for the MIT licensed files:
-
-```
-The MIT License (MIT)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-```
-
-## The rest of this project is in the worldwide public domain
-
-As a work of the United States government, this project is in the public domain within the United States.
-
-Additionally, we waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
-
-### CC0 1.0 Universal Summary
-
-This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode).
-
-#### No copyright
-
-The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law.
-
-You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.
-
-#### Other information
-
-In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights.
-
-Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer.
-
-## Contributions to this project
-
-As stated in [CONTRIBUTING](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.
-
-## Trademark
-
-GSA owns, reserves, and retains all rights, title, and interest in all trademarks owned by GSA. The User does not acquire any right or interest of any kind in any GSA trademark because of their use of the Modified Version or any of the modifications made to the Work by GSA.

Makefile

@@ -7,100 +7,192 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
-## DEVELOPMENT
+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
+DOCKER_TTY ?= $(if ${JENKINS_HOME},,t)
 
-.PHONY: bootstrap
-bootstrap: ## Set up everything to run the app
-	make generate-version-file
-	pipenv install --dev
-	createdb notification_api || true
-	(pipenv run flask db upgrade) || true
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
 
-.PHONY: bootstrap-with-docker
-bootstrap-with-docker: ## Build the image to run the app in Docker
-	docker build -f docker/Dockerfile -t notifications-api .
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
 
-.PHONY: run-procfile
-run-procfile:
-	pipenv run honcho start -f Procfile.dev
+CF_API ?= api.cloud.service.gov.uk
+CF_ORG ?= govuk-notify
+CF_SPACE ?= ${DEPLOY_ENV}
+CF_HOME ?= ${HOME}
+$(eval export CF_HOME)
 
-.PHONY: run-flask
-run-flask: ## Run flask
-	pipenv run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
 
-.PHONY: run-celery
-run-celery: ## Run celery, TODO remove purge for staging/prod
-	pipenv run celery -A run_celery.notify_celery purge -f
-	pipenv run newrelic-admin run-program celery \
-		-A run_celery.notify_celery worker \
-		--pidfile="/tmp/celery.pid" \
-		--loglevel=INFO \
-		--concurrency=4
-
-.PHONY: run-celery-beat
-run-celery-beat: ## Run celery beat
-	pipenv run celery \
-	-A run_celery.notify_celery beat \
-	--loglevel=INFO
+NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+.PHONY: check-env-vars
+check-env-vars: ## Check mandatory environment variables
+	$(if ${DEPLOY_ENV},,$(error Must specify DEPLOY_ENV))
+
+.PHONY: preview
+preview: ## Set environment to preview
+	$(eval export DEPLOY_ENV=preview)
+	@true
+
+.PHONY: staging
+staging: ## Set environment to staging
+	$(eval export DEPLOY_ENV=staging)
+	@true
+
+.PHONY: production
+production: ## Set environment to production
+	$(eval export DEPLOY_ENV=production)
+	@true
+
 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
-	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
+	@echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE}
+
+.PHONY: build-paas-artifact
+build-paas-artifact:  ## Build the deploy artifact for PaaS
+	rm -rf target
+	mkdir -p target
+	zip -y -q -r -x@deploy-exclude.lst target/notifications-api.zip ./
+
+.PHONY: upload-paas-artifact
+upload-paas-artifact: ## Upload the deploy artifact for PaaS
+	$(if ${DEPLOY_BUILD_NUMBER},,$(error Must specify DEPLOY_BUILD_NUMBER))
+	$(if ${JENKINS_S3_BUCKET},,$(error Must specify JENKINS_S3_BUCKET))
+	aws s3 cp --region eu-west-1 --sse AES256 target/notifications-api.zip s3://${JENKINS_S3_BUCKET}/build/notifications-api/${DEPLOY_BUILD_NUMBER}.zip
 
 .PHONY: test
-test: export NEW_RELIC_ENVIRONMENT=test
-test: ## Run tests
-	pipenv run flake8 .
-	pipenv run isort --check-only ./app ./tests
-	pipenv run pytest -n4 --maxfail=10
+test: generate-version-file ## Run tests
+	./scripts/run_tests.sh
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	pipenv lock
-	pipenv requirements
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
 
-.PHONY: audit
-audit:
-	pipenv requirements > requirements.txt
-	pipenv requirements --dev > requirements_for_test.txt
-	pipenv run pip-audit -r requirements.txt
-	-pipenv run pip-audit -r requirements_for_test.txt
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
 
-.PHONY: static-scan
-static-scan:
-	pipenv run bandit -r app/
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
+	docker build -f docker/Dockerfile \
+		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
+		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
+		--build-arg NO_PROXY="${NO_PROXY}" \
+		-t ${DOCKER_BUILDER_IMAGE_NAME} \
+		.
+
+.PHONY: build-with-docker
+build-with-docker: ; ## don't do anything
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	@docker run -i${DOCKER_TTY} --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
+		-e GIT_COMMIT=${GIT_COMMIT} \
+		-e BUILD_NUMBER=${BUILD_NUMBER} \
+		-e BUILD_URL=${BUILD_URL} \
+		-e http_proxy="${HTTP_PROXY}" \
+		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e https_proxy="${HTTPS_PROXY}" \
+		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e NO_PROXY="${NO_PROXY}" \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: create-docker-test-db
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	@docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
 
 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
+	rm -rf node_modules cache target venv .coverage build tests/.cache
+
+.PHONY: cf-login
+cf-login: ## Log in to Cloud Foundry
+	$(if ${CF_USERNAME},,$(error Must specify CF_USERNAME))
+	$(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	@echo "Logging in to Cloud Foundry on ${CF_API}"
+	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
+
+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@jinja2 --strict manifest.yml.j2 \
+	    -D environment=${CF_SPACE} \
+	    -D CF_APP=${CF_APP} \
+	    --format=yaml \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
+
+.PHONY: cf-deploy
+cf-deploy: ## Deploys the app to Cloud Foundry
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	cf target -o ${CF_ORG} -s ${CF_SPACE}
+	@cf app --guid ${CF_APP} || exit 1
+
+	# cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment)
+	cf v3-cancel-zdt-push ${CF_APP} || true
+
+	cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest)
+	CF_STARTUP_TIMEOUT=10 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete  # fails after 5 mins if deploy doesn't work
 
 
-## DEPLOYMENT
+.PHONY: cf-deploy-api-db-migration
+cf-deploy-api-db-migration:
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	cf target -o ${CF_ORG} -s ${CF_SPACE}
+	cf push notify-api-db-migration --no-route -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
+	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
 
-# .PHONY: cf-deploy-failwhale
-# cf-deploy-failwhale:
-# 	$(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`))
-# 	cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml
+.PHONY: cf-check-api-db-migration-task
+cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
+	@cf curl /v3/apps/`cf app --guid notify-api-db-migration`/tasks?order_by=-created_at | jq -r ".resources[0].state"
 
-# .PHONY: enable-failwhale
-# enable-failwhale: ## Enable the failwhale app and disable api
-# 	$(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`))
-# 	# make sure failwhale is running first
-# 	cf start notify-api-failwhale
+.PHONY: cf-rollback
+cf-rollback: ## Rollbacks the app to the previous release
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	cf v3-cancel-zdt-push ${CF_APP}
 
-# 	cf map-route notify-api-failwhale ${DNS_NAME} --hostname api
-# 	cf unmap-route notify-api ${DNS_NAME} --hostname api
-# 	@echo "Failwhale is enabled"
+.PHONY: cf-push
+cf-push:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	cf target -o ${CF_ORG} -s ${CF_SPACE}
+	cf push ${CF_APP} -f <(make -s generate-manifest)
 
-# .PHONY: disable-failwhale
-# disable-failwhale: ## Disable the failwhale app and enable api
-# 	$(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`))
-
-# 	cf map-route notify-api ${DNS_NAME} --hostname api
-# 	cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api
-# 	cf stop notify-api-failwhale
-# 	@echo "Failwhale is disabled"
+.PHONY: check-if-migrations-to-run
+check-if-migrations-to-run:
+	@echo $(shell python3 scripts/check_if_new_migration.py)

Pipfile

@@ -1,83 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-alembic = "==1.7.7"
-amqp = "==5.1.1"
-arrow = "==1.2.2"
-asn1crypto = "==1.5.1"
-async-timeout = "==4.0.2"
-attrs = "==21.4.0"
-awscli = "==1.24.8"
-bcrypt = "==3.2.2"
-beautifulsoup4 = "==4.11.1"
-billiard = "==3.6.4.0"
-bleach = "==4.1.0"
-blinker = "==1.4"
-boto3 = "==1.23.8"
-botocore = "==1.26.8"
-cachetools = "==5.1.0"
-celery = {version = "==5.2.7", extras = ["redis"]}
-certifi = ">=2022.12.7"
-cffi = "==1.15.0"
-charset-normalizer = "==2.0.12"
-click = "==8.1.3"
-click-datetime = "==0.2"
-click-didyoumean = "==0.3.0"
-click-plugins = "==1.1.1"
-click-repl = "==0.2.0"
-colorama = "==0.4.4"
-defusedxml = "==0.7.1"
-deprecated = "==1.2.13"
-dnspython = "==2.2.1"
-docopt = "==0.6.2"
-docutils = "==0.16"
-eventlet = "==0.33.1"
-flask = "~=2.2"
-flask-bcrypt = "==1.0.1"
-flask-marshmallow = "==0.14.0"
-flask-migrate = "==3.1.0"
-flask-redis = "==0.4.0"
-flask-sqlalchemy = {version = "==2.5.1", ref = "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", git = "https://github.com/pallets-eco/flask-sqlalchemy.git"}
-gunicorn = {version = "==20.1.0", extras = ["eventlet"], ref = "1299ea9e967a61ae2edebe191082fd169b864c64", git = "https://github.com/benoitc/gunicorn.git"}
-iso8601 = "==1.0.2"
-itsdangerous = "==2.1.2"
-jsonschema = {version = "==4.5.1", extras = ["format"]}
-lxml = "==4.9.1"
-marshmallow = "==3.15.0"
-marshmallow-sqlalchemy = "==0.28.1"
-notifications-python-client = "==6.3.0"
-oscrypto = "==1.3.0"
-psycopg2-binary = "==2.9.3"
-pyjwt = "==2.4.0"
-python-dotenv = "==0.20.0"
-sqlalchemy = "==1.4.40"
-werkzeug = "~=2.2"
-# gds metrics packages
-prometheus-client = "==0.14.1"
-gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
-packaging = "==21.3"
-notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
-newrelic = "*"
-
-[dev-packages]
-flake8 = "==4.0.1"
-flake8-bugbear = "==22.4.25"
-isort = "==5.10.1"
-moto = "==3.1.9"
-pytest = "==7.1.2"
-pytest-env = "==0.6.2"
-pytest-mock = "==3.7.0"
-pytest-cov = "==3.0.0"
-pytest-xdist = "==2.5.0"
-freezegun = "==1.2.1"
-requests-mock = "==1.9.3"
-jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
-pip-audit = "*"
-bandit = "*"
-honcho = "*"
-
-[requires]
-python_version = "3.9"

Procfile

@@ -0,0 +1 @@
+web: ./scripts/paas_app_wrapper.sh

Procfile.dev

@@ -1,3 +0,0 @@
-web: make run-flask
-worker: make run-celery
-scheduler: make run-celery-beat

README.md

@@ -1,120 +1,158 @@
-# US Notify API
+# GOV.UK Notify API
 
-This project is the core of [Notify](https://notifications-admin.app.cloud.gov/). It's cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers!
+Contains:
+- the public-facing REST API for GOV.UK Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation)
+- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/alphagov/notifications-admin) talks to)
+- asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc
 
-This repo contains:
+## Setting Up
 
-- A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation)
-- An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to)
-- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc.
+### Python version
 
-Our other repositories are:
+At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer.
 
-- [notifications-admin](https://github.com/GSA/notifications-admin)
-- [notifications-utils](https://github.com/GSA/notifications-utils)
-- [us-notify-compliance](https://github.com/GSA/us-notify-compliance/)
-- [notify-python-demo](https://github.com/GSA/notify-python-demo)
+### AWS credentials
 
-## Documentation, here and elsewhere
+To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.
 
-### About Notify
+Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
 
-- [Roadmap](https://notifications-admin.app.cloud.gov/features/roadmap)
-- [Using the API](./docs/api-usage.md)
+### Virtualenv
 
-### Infrastructure
+```
+mkvirtualenv -p /usr/local/bin/python3 notifications-api
+```
 
-- [Overview, setup, and onboarding](./docs/infra-overview.md)
-- [Database management](./docs/database-management.md)
-- [Celery queues and tasks](./docs/queues-and-tasks.md)
+### `environment.sh`
 
-### Common dev work
+Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
 
-- [Local setup](#local-setup)
-- [Testing](./docs/testing.md), both automated and manual
-- [Deploying](./docs/deploying.md)
-- [Running one-off tasks](./docs/one-off-tasks.md)
+Create a local environment.sh file containing the following:
 
-## UK docs that may still be helpful
+```
+echo "
+export NOTIFY_ENVIRONMENT='development'
 
-- [Writing public APIs](docs/writing-public-apis.md)
-- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies)
+export MMG_API_KEY='MMG_API_KEY'
+export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
+export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
 
-## Local setup
+export FLASK_APP=application.py
+export FLASK_DEBUG=1
+export WERKZEUG_DEBUG_PIN=off
+"> environment.sh
+```
 
-### Common steps
+NOTES:
 
-1. Install pre-requisites for setup:
-    * [jq](https://stedolan.github.io/jq/): `brew install jq`
-    * [terraform](https://www.terraform.io/): `brew install terraform` or `brew install tfenv` and use `tfenv` to install `terraform ~> 1.4.0`
-    * [cf-cli@8](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html): `brew install cloudfoundry/tap/cf-cli@8`
-1. [Log into cloud.gov](https://cloud.gov/docs/getting-started/setup/#set-up-the-command-line): `cf login -a api.fr.cloud.gov --sso`
-1. Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov
-1. Run the development terraform with:
+ * Replace the placeholder key and prefix values as appropriate
+ * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
+ * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.
 
-        ```
-        $ cd terraform/development
-        $ ./run.sh
-        ```
+### Postgres
 
-1. If you want to send data to New Relic from your local develpment environment, set `NEW_RELIC_LICENSE_KEY` within `.env`
-1. Follow the instructions for either `Direct installation` or `Docker installation` below
+Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
 
-### Direct installation
+Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.
 
-1. Set up Postgres && Redis on your machine
+### Redis
 
-1. Install [pipenv](https://pipenv.pypa.io/en/latest/)
+To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
 
-1. Run the project setup
-
-    `make bootstrap`
-
-1. Run the web server and background worker
-
-    `make run-procfile`
-
-1. Or run them individually:
-
-    * Run Flask (web server)
-
-        `make run-flask`
-
-    * Run Celery (background worker)
-
-        `make run-celery`
+        REDIS_ENABLED = True
 
 
-### VS Code && Docker installation
+##  To run the application
 
-If you're working in VS Code, you can also leverage Docker for a containerized dev environment
+First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
 
-1. Uncomment the `Local Docker setup` lines in `.env` and comment out the `Local direct setup` lines.
+You need to run the api application and a local celery instance.
 
-1. Install the Remote-Containers plug-in in VS Code
+There are two run scripts for running all the necessary parts.
 
-1. With Docker running, create the network:
+```
+scripts/run_app.sh
+```
 
-    `docker network create notify-network`
+```
+scripts/run_celery.sh
+```
 
-1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select “Remote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notifications-api). This will start the container in a new window, replacing the current one.
+Optionally you can also run this script to run the scheduled tasks:
 
-1. Wait a few minutes while things happen 🍵
+```
+scripts/run_celery_beat.sh
+```
 
-1. Open a VS Code terminal and run the Flask application:
 
-    `make run-flask`
+##  To test the application
 
-1. Open another VS Code terminal and run Celery:
+First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
 
-    `make run-celery`
+Then simply run
 
-NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. VS Code _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container.
+```
+make test
+```
 
-## License && public domain
+That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
+[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
 
-Work through [commit `e604385`](https://github.com/GSA/notifications-api/commit/e604385e0cf4c2ab8c6451b7120ceb196cce21b5) is licensed by the UK government under the MIT license. Work after that commit is in the worldwide public domain. See [LICENSE.md](./LICENSE.md) for more information.
 
-## Contributing
+## To update application dependencies
 
-As stated in [CONTRIBUTING.md](CONTRIBUTING.md), all contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with
+
+```
+make freeze-requirements
+```
+
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.
+
+
+## To run one off tasks
+
+Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
+care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
+example, to purge all dynamically generated functional test data, do the following:
+
+Locally
+```
+flask command purge_functional_test_data -u <functional tests user name prefix>
+```
+
+On the server
+```
+cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
+```
+
+All commands and command options have a --help command if you need more information.
+
+
+## To create a new worker app
+
+You need to:
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
+1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
+1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
+1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))
+
+**Important:**
+
+Before pushing the deployment change on jenkins, read below about the first time deployment.
+
+### First time deployment of your new worker
+
+Our deployment flow requires that the app is present in order to proceed with the deployment.
+
+This means that the first deployment of your app must happen manually.
+
+To do this:
+
+1. Ensure your code is backwards compatible
+1. From the root of this repo run `CF_APP=<APP_NAME> make <cf-space> cf-push`
+
+Once this is done, you can push your deployment changes to jenkins to have your app deployed on every deployment.

app/__init__.py

@@ -1,77 +1,64 @@
 import os
-import secrets
+import random
 import string
-import time
 import uuid
-from time import monotonic
 
-from celery import current_task
-from flask import (
-    current_app,
-    g,
-    has_request_context,
-    jsonify,
-    make_response,
-    request,
-)
+from flask import _request_ctx_stack, request, g, jsonify, make_response
+from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
-from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
-from gds_metrics import GDSMetrics
-from gds_metrics.metrics import Gauge, Histogram
-from notifications_utils import logging, request_helper
-from notifications_utils.celery import NotifyCelery
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from time import monotonic
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
-from sqlalchemy import event
+from notifications_utils.clients.statsd.statsd_client import StatsdClient
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils import logging, request_helper
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy
 
-from app.clients import NotificationProviderClients
+from app.celery.celery import NotifyCelery
+from app.clients import Clients
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
-from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.aws_sns import AwsSnsClient
+from app.clients.sms.firetext import FiretextClient
+from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
+
+DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f"
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+DATE_FORMAT = "%Y-%m-%d"
 
 
 class SQLAlchemy(_SQLAlchemy):
     """We need to subclass SQLAlchemy in order to override create_engine options"""
 
     def apply_driver_hacks(self, app, info, options):
-        sa_url, options = super().apply_driver_hacks(app, info, options)
+        super().apply_driver_hacks(app, info, options)
         if 'connect_args' not in options:
             options['connect_args'] = {}
         options['connect_args']["options"] = "-c statement_timeout={}".format(
             int(app.config['SQLALCHEMY_STATEMENT_TIMEOUT']) * 1000
         )
-        return (sa_url, options)
 
 
 db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
+firetext_client = FiretextClient()
+mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
-aws_ses_stub_client = AwsSesStubClient()
-aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
+performance_platform_client = PerformancePlatformClient()
 document_download_client = DocumentDownloadClient()
-metrics = GDSMetrics()
 
-notification_provider_clients = NotificationProviderClients()
+clients = Clients()
 
-api_user = LocalProxy(lambda: g.api_user)
-authenticated_service = LocalProxy(lambda: g.authenticated_service)
-
-CONCURRENT_REQUESTS = Gauge(
-    'concurrent_web_request_count',
-    'How many concurrent requests are currently being served',
-)
+api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
+authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
 
 
 def create_app(application):
@@ -83,34 +70,22 @@ def create_app(application):
 
     application.config['NOTIFY_APP_NAME'] = application.name
     init_app(application)
-
-    # Metrics intentionally high up to give the most accurate timing and reliability that the metric is recorded
-    metrics.init_app(application)
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)
     ma.init_app(application)
     zendesk_client.init_app(application)
     statsd_client.init_app(application)
-    logging.init_app(application)
-    aws_sns_client.init_app(application, statsd_client=statsd_client)
-
-    aws_ses_client.init_app(statsd_client=statsd_client)
-    aws_ses_stub_client.init_app(
-        statsd_client=statsd_client,
-        stub_url=application.config['SES_STUB_URL']
-    )
-    # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
-    email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
-    notification_provider_clients.init_app(
-        sms_clients=[aws_sns_client],
-        email_clients=email_clients
-    )
-
+    logging.init_app(application, statsd_client)
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application, statsd_client=statsd_client)
+    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     notify_celery.init_app(application)
     encryption.init_app(application)
     redis_store.init_app(application)
+    performance_platform_client.init_app(application)
     document_download_client.init_app(application)
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client])
 
     register_blueprint(application)
     register_v2_blueprints(application)
@@ -119,54 +94,38 @@ def create_app(application):
     from app.commands import setup_commands
     setup_commands(application)
 
-    # set up sqlalchemy events
-    setup_sqlalchemy_events(application)
-
     return application
 
 
 def register_blueprint(application):
-    from app.authentication.auth import (
-        requires_admin_auth,
-        requires_auth,
-        requires_no_auth,
-    )
-    from app.billing.rest import billing_blueprint
-    from app.complaint.complaint_rest import complaint_blueprint
-    from app.docs import docs as docs_blueprint
-    from app.email_branding.rest import email_branding_blueprint
+    from app.service.rest import service_blueprint
+    from app.service.callback_rest import service_callback_blueprint
+    from app.user.rest import user_blueprint
+    from app.template.rest import template_blueprint
+    from app.status.healthcheck import status as status_blueprint
+    from app.job.rest import job_blueprint
+    from app.notifications.rest import notifications as notifications_blueprint
+    from app.invite.rest import invite as invite_blueprint
+    from app.accept_invite.rest import accept_invite
+    from app.template_statistics.rest import template_statistics as template_statistics_blueprint
     from app.events.rest import events as events_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
+    from app.email_branding.rest import email_branding_blueprint
     from app.inbound_number.rest import inbound_number_blueprint
     from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
-    from app.job.rest import job_blueprint
-    from app.notifications.notifications_ses_callback import (
-        ses_callback_blueprint,
-    )
-    from app.notifications.receive_notifications import (
-        receive_notifications_blueprint,
-    )
-    from app.notifications.rest import notifications as notifications_blueprint
-    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.notifications.receive_notifications import receive_notifications_blueprint
+    from app.notifications.notifications_sms_callback import sms_callback_blueprint
+    from app.notifications.notifications_letter_callback import letter_callback_blueprint
+    from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth
+    from app.letters.rest import letter_job
+    from app.billing.rest import billing_blueprint
     from app.organisation.rest import organisation_blueprint
-    from app.performance_dashboard.rest import performance_dashboard_blueprint
+    from app.organisation.invite_rest import organisation_invite_blueprint
+    from app.complaint.complaint_rest import complaint_blueprint
     from app.platform_stats.rest import platform_stats_blueprint
-    from app.provider_details.rest import (
-        provider_details as provider_details_blueprint,
-    )
-    from app.service.callback_rest import service_callback_blueprint
-    from app.service.rest import service_blueprint
-    from app.service_invite.rest import (
-        service_invite as service_invite_blueprint,
-    )
-    from app.status.healthcheck import status as status_blueprint
-    from app.template.rest import template_blueprint
     from app.template_folder.rest import template_folder_blueprint
-    from app.template_statistics.rest import (
-        template_statistics as template_statistics_blueprint,
-    )
+    from app.letter_branding.letter_branding_rest import letter_branding_blueprint
     from app.upload.rest import upload_blueprint
-    from app.user.rest import user_blueprint
-    from app.webauthn.rest import webauthn_blueprint
 
     service_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(service_blueprint, url_prefix='/service')
@@ -174,21 +133,16 @@ def register_blueprint(application):
     user_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(user_blueprint, url_prefix='/user')
 
-    webauthn_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(webauthn_blueprint)
-
     template_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_blueprint)
 
     status_blueprint.before_request(requires_no_auth)
     application.register_blueprint(status_blueprint)
 
-    docs_blueprint.before_request(requires_no_auth)
-    application.register_blueprint(docs_blueprint)
-
     # delivery receipts
-    ses_callback_blueprint.before_request(requires_no_auth)
-    application.register_blueprint(ses_callback_blueprint)
+    # TODO: make sure research mode can still trigger sms callbacks, then re-enable this
+    sms_callback_blueprint.before_request(requires_no_auth)
+    application.register_blueprint(sms_callback_blueprint)
 
     # inbound sms
     receive_notifications_blueprint.before_request(requires_no_auth)
@@ -200,11 +154,8 @@ def register_blueprint(application):
     job_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(job_blueprint)
 
-    service_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(service_invite_blueprint)
-
-    organisation_invite_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(organisation_invite_blueprint)
+    invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(invite_blueprint)
 
     inbound_number_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_number_blueprint)
@@ -212,6 +163,9 @@ def register_blueprint(application):
     inbound_sms_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(inbound_sms_blueprint)
 
+    accept_invite.before_request(requires_admin_auth)
+    application.register_blueprint(accept_invite, url_prefix='/invite')
+
     template_statistics_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_statistics_blueprint)
 
@@ -224,6 +178,12 @@ def register_blueprint(application):
     email_branding_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(email_branding_blueprint, url_prefix='/email-branding')
 
+    letter_job.before_request(requires_admin_auth)
+    application.register_blueprint(letter_job)
+
+    letter_callback_blueprint.before_request(requires_no_auth)
+    application.register_blueprint(letter_callback_blueprint)
+
     billing_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(billing_blueprint)
 
@@ -233,63 +193,65 @@ def register_blueprint(application):
     organisation_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(organisation_blueprint, url_prefix='/organisations')
 
+    organisation_invite_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(organisation_invite_blueprint)
+
     complaint_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(complaint_blueprint)
 
-    performance_dashboard_blueprint.before_request(requires_admin_auth)
-    application.register_blueprint(performance_dashboard_blueprint)
-
     platform_stats_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats')
 
     template_folder_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(template_folder_blueprint)
 
+    letter_branding_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(letter_branding_blueprint)
+
     upload_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(upload_blueprint)
 
 
 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
+    from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
+    from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
+    from app.v2.template.get_template import v2_template_blueprint as get_template
+    from app.v2.templates.get_templates import v2_templates_blueprint as get_templates
+    from app.v2.template.post_template import v2_template_blueprint as post_template
     from app.authentication.auth import requires_auth
-    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint
-    from app.v2.notifications import (  # noqa
-        get_notifications,
-        post_notifications,
-        v2_notification_blueprint,
-    )
-    from app.v2.template import (  # noqa
-        get_template,
-        post_template,
-        v2_template_blueprint,
-    )
-    from app.v2.templates.get_templates import v2_templates_blueprint
 
-    v2_notification_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_notification_blueprint)
+    post_notifications.before_request(requires_auth)
+    application.register_blueprint(post_notifications)
 
-    v2_templates_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_templates_blueprint)
+    get_notifications.before_request(requires_auth)
+    application.register_blueprint(get_notifications)
 
-    v2_template_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_template_blueprint)
+    get_templates.before_request(requires_auth)
+    application.register_blueprint(get_templates)
 
-    v2_inbound_sms_blueprint.before_request(requires_auth)
-    application.register_blueprint(v2_inbound_sms_blueprint)
+    get_template.before_request(requires_auth)
+    application.register_blueprint(get_template)
+
+    post_template.before_request(requires_auth)
+    application.register_blueprint(post_template)
+
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)
 
 
 def init_app(app):
+    @app.before_request
+    def record_user_agent():
+        statsd_client.incr("user-agent.{}".format(process_user_agent(request.headers.get('User-Agent', None))))
 
     @app.before_request
     def record_request_details():
-        CONCURRENT_REQUESTS.inc()
-
         g.start = monotonic()
         g.endpoint = request.endpoint
 
     @app.after_request
     def after_request(response):
-        CONCURRENT_REQUESTS.dec()
-
         response.headers.add('Access-Control-Allow-Origin', '*')
         response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
         response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE')
@@ -322,90 +284,16 @@ def create_uuid():
 
 
 def create_random_identifier():
-    return ''.join(secrets.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
 
 
-def setup_sqlalchemy_events(app):
-
-    TOTAL_DB_CONNECTIONS = Gauge(
-        'db_connection_total_connected',
-        'How many db connections are currently held (potentially idle) by the server',
-    )
-
-    TOTAL_CHECKED_OUT_DB_CONNECTIONS = Gauge(
-        'db_connection_total_checked_out',
-        'How many db connections are currently checked out by web requests',
-    )
-
-    DB_CONNECTION_OPEN_DURATION_SECONDS = Histogram(
-        'db_connection_open_duration_seconds',
-        'How long db connections are held open for in seconds',
-        ['method', 'host', 'path']
-    )
-
-    # need this or db.engine isn't accessible
-    with app.app_context():
-        @event.listens_for(db.engine, 'connect')
-        def connect(dbapi_connection, connection_record):
-            # connection first opened with db
-            TOTAL_DB_CONNECTIONS.inc()
-
-        @event.listens_for(db.engine, 'close')
-        def close(dbapi_connection, connection_record):
-            # connection closed (probably only happens with overflow connections)
-            TOTAL_DB_CONNECTIONS.dec()
-
-        @event.listens_for(db.engine, 'checkout')
-        def checkout(dbapi_connection, connection_record, connection_proxy):
-            try:
-                # connection given to a web worker
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.inc()
-
-                # this will overwrite any previous checkout_at timestamp
-                connection_record.info['checkout_at'] = time.monotonic()
-
-                # checkin runs after the request is already torn down, therefore we add the request_data onto the
-                # connection_record as otherwise it won't have that information when checkin actually runs.
-                # Note: this is not a problem for checkouts as the checkout always happens within a web request or task
-
-                # web requests
-                if has_request_context():
-                    connection_record.info['request_data'] = {
-                        'method': request.method,
-                        'host': request.host,
-                        'url_rule': request.url_rule.rule if request.url_rule else 'No endpoint'
-                    }
-                # celery apps
-                elif current_task:
-                    connection_record.info['request_data'] = {
-                        'method': 'celery',
-                        'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
-                        'url_rule': current_task.name,  # task name
-                    }
-                # anything else. migrations possibly, or flask cli commands.
-                else:
-                    current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
-                    connection_record.info['request_data'] = {
-                        'method': 'unknown',
-                        'host': 'unknown',
-                        'url_rule': 'unknown',
-                    }
-            except Exception:
-                current_app.logger.exception("Exception caught for checkout event.")
-
-        @event.listens_for(db.engine, 'checkin')
-        def checkin(dbapi_connection, connection_record):
-            try:
-                # connection returned by a web worker
-                TOTAL_CHECKED_OUT_DB_CONNECTIONS.dec()
-
-                # duration that connection was held by a single web request
-                duration = time.monotonic() - connection_record.info['checkout_at']
-
-                DB_CONNECTION_OPEN_DURATION_SECONDS.labels(
-                    connection_record.info['request_data']['method'],
-                    connection_record.info['request_data']['host'],
-                    connection_record.info['request_data']['url_rule']
-                ).observe(duration)
-            except Exception:
-                current_app.logger.exception("Exception caught for checkin event.")
+def process_user_agent(user_agent_string):
+    if user_agent_string and user_agent_string.lower().startswith("notify"):
+        components = user_agent_string.split("/")
+        client_name = components[0].lower()
+        client_version = components[1].replace(".", "-")
+        return "{}.{}".format(client_name, client_version)
+    elif user_agent_string and not user_agent_string.lower().startswith("notify"):
+        return "non-notify-user-agent"
+    else:
+        return "unknown"

app/accept_invite/rest.py

@@ -0,0 +1,52 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    current_app
+)
+
+from itsdangerous import SignatureExpired, BadData
+
+from notifications_utils.url_safe_token import check_token
+
+from app.dao.invited_user_dao import get_invited_user_by_id
+from app.dao.organisation_dao import dao_get_invited_organisation_user
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
+from app.schemas import invited_user_schema
+
+
+accept_invite = Blueprint('accept_invite', __name__)
+register_errors(accept_invite)
+
+
+@accept_invite.route('/<invitation_type>/<token>', methods=['GET'])
+def validate_invitation_token(invitation_type, token):
+
+    max_age_seconds = 60 * 60 * 24 * current_app.config['INVITATION_EXPIRATION_DAYS']
+
+    try:
+        invited_user_id = check_token(token,
+                                      current_app.config['SECRET_KEY'],
+                                      current_app.config['DANGEROUS_SALT'],
+                                      max_age_seconds)
+    except SignatureExpired:
+        errors = {'invitation':
+                  ['Your invitation to GOV.UK Notify has expired. '
+                   'Please ask the person that invited you to send you another one']}
+        raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
+
+    if invitation_type == 'service':
+        invited_user = get_invited_user_by_id(invited_user_id)
+        return jsonify(data=invited_user_schema.dump(invited_user).data), 200
+    elif invitation_type == 'organisation':
+        invited_user = dao_get_invited_organisation_user(invited_user_id)
+        return jsonify(data=invited_user.serialize()), 200
+    else:
+        raise InvalidRequest("Unrecognised invitation type: {}".format(invitation_type))

app/authentication/auth.py

@@ -1,33 +1,13 @@
-import uuid
-
-from flask import current_app, g, request
-from gds_metrics import Histogram
-from notifications_python_client.authentication import (
-    decode_jwt_token,
-    get_token_issuer,
-)
+from flask import request, _request_ctx_stack, current_app, g
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
 from notifications_python_client.errors import (
-    TokenAlgorithmError,
-    TokenDecodeError,
-    TokenError,
-    TokenExpiredError,
-    TokenIssuerError,
+    TokenDecodeError, TokenExpiredError, TokenIssuerError, TokenAlgorithmError
 )
 from notifications_utils import request_helper
+from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 
-from app.serialised_models import SerialisedService
-
-# stvnrlly - this is silly, but bandit has a multiline string bug (https://github.com/PyCQA/bandit/issues/658)
-# and flake8 wants a multiline quote here. TODO: check on bug status and restore sanity once possible
-TOKEN_MESSAGE_ONE = "Invalid token: make sure your API token matches the example "  # nosec B105
-TOKEN_MESSAGE_TWO = "at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header"  # nosec B105
-GENERAL_TOKEN_ERROR_MESSAGE = TOKEN_MESSAGE_ONE + TOKEN_MESSAGE_TWO
-
-AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram(
-    'auth_db_connection_duration_seconds',
-    'Time taken to get DB connection and fetch service from database',
-)
+from app.dao.services_dao import dao_fetch_service_by_id_with_api_keys
 
 
 class AuthError(Exception):
@@ -53,11 +33,17 @@ class AuthError(Exception):
         }
 
 
-class InternalApiKey():
-    def __init__(self, client_id, secret):
-        self.secret = secret
-        self.id = client_id
-        self.expiry_date = None
+def get_auth_token(req):
+    auth_header = req.headers.get('Authorization', None)
+    if not auth_header:
+        raise AuthError('Unauthorized, authentication token must be provided', 401)
+
+    auth_scheme = auth_header[:7].title()
+
+    if auth_scheme != 'Bearer ':
+        raise AuthError('Unauthorized, authentication bearer scheme must be used', 401)
+
+    return auth_header[7:]
 
 
 def requires_no_auth():
@@ -65,45 +51,28 @@ def requires_no_auth():
 
 
 def requires_admin_auth():
-    requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID'))
-
-
-def requires_internal_auth(expected_client_id):
-    if expected_client_id not in current_app.config.get('INTERNAL_CLIENT_API_KEYS'):
-        raise TypeError("Unknown client_id for internal auth")
-
     request_helper.check_proxy_header_before_request()
-    auth_token = _get_auth_token(request)
-    client_id = _get_token_issuer(auth_token)
 
-    if client_id != expected_client_id:
-        current_app.logger.info('client_id: %s', client_id)
-        current_app.logger.info('expected_client_id: %s', expected_client_id)
-        raise AuthError("Unauthorized: not allowed to perform this action", 401)
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
 
-    api_keys = [
-        InternalApiKey(client_id, secret)
-        for secret in current_app.config.get('INTERNAL_CLIENT_API_KEYS')[client_id]
-    ]
-
-    _decode_jwt_token(auth_token, api_keys, client_id)
-    g.service_id = client_id
+    if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
+        g.service_id = current_app.config.get('ADMIN_CLIENT_USER_NAME')
+        return handle_admin_key(auth_token, current_app.config.get('ADMIN_CLIENT_SECRET'))
+    else:
+        raise AuthError('Unauthorized, admin authentication token required', 401)
 
 
 def requires_auth():
     request_helper.check_proxy_header_before_request()
 
-    auth_token = _get_auth_token(request)
-    issuer = _get_token_issuer(auth_token)  # ie the `iss` claim which should be a service ID
+    auth_token = get_auth_token(request)
+    client = __get_token_issuer(auth_token)
 
     try:
-        service_id = uuid.UUID(issuer)
-    except Exception:
+        service = dao_fetch_service_by_id_with_api_keys(client)
+    except DataError:
         raise AuthError("Invalid token: service id is not the right data type", 403)
-
-    try:
-        with AUTH_DB_CONNECTION_DURATION_SECONDS.time():
-            service = SerialisedService.from_id(service_id)
     except NoResultFound:
         raise AuthError("Invalid token: service not found", 403)
 
@@ -113,69 +82,49 @@ def requires_auth():
     if not service.active:
         raise AuthError("Invalid token: service is archived", 403, service_id=service.id)
 
-    api_key = _decode_jwt_token(auth_token, service.api_keys, service.id)
-
-    current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format(
-        service_id,
-        api_key.id,
-        request.headers.get('User-Agent'),
-        request.base_url
-    ))
-
-    g.api_user = api_key
-    g.service_id = service_id
-    g.authenticated_service = service
-
-
-def _decode_jwt_token(auth_token, api_keys, service_id=None):
-    for api_key in api_keys:
+    for api_key in service.api_keys:
         try:
             decode_jwt_token(auth_token, api_key.secret)
         except TokenExpiredError:
-            if not current_app.config.get('ALLOW_EXPIRED_API_TOKEN', False):
-                err_msg = "Error: Your system clock must be accurate to within 30 seconds"
-                raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
+            err_msg = "Error: Your system clock must be accurate to within 30 seconds"
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
         except TokenAlgorithmError:
             err_msg = "Invalid token: algorithm used is not HS256"
-            raise AuthError(err_msg, 403, service_id=service_id, api_key_id=api_key.id)
+            raise AuthError(err_msg, 403, service_id=service.id, api_key_id=api_key.id)
         except TokenDecodeError:
-            # we attempted to validate the token but it failed meaning it was not signed using this api key.
-            # Let's try the next one
-            # TODO: Change this so it doesn't also catch `TokenIssuerError` or `TokenIssuedAtError` exceptions (which
-            # are children of `TokenDecodeError`) as these should cause an auth error immediately rather than
-            # continue on to check the next API key
             continue
-        except TokenError:
-            # General error when trying to decode and validate the token
-            raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403, service_id=service_id, api_key_id=api_key.id)
 
         if api_key.expiry_date:
-            raise AuthError("Invalid token: API key revoked", 403, service_id=service_id, api_key_id=api_key.id)
+            raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id)
 
-        return api_key
+        g.service_id = api_key.service_id
+        _request_ctx_stack.top.authenticated_service = service
+        _request_ctx_stack.top.api_user = api_key
+        current_app.logger.info('API authorised for service {} with api key {}, using client {}'.format(
+            service.id,
+            api_key.id,
+            request.headers.get('User-Agent')
+        ))
+        return
     else:
         # service has API keys, but none matching the one the user provided
-        raise AuthError("Invalid token: API key not found", 403, service_id=service_id)
+        raise AuthError("Invalid token: signature, api token not found", 403, service_id=service.id)
 
 
-def _get_auth_token(req):
-    auth_header = req.headers.get('Authorization', None)
-    if not auth_header:
-        raise AuthError('Unauthorized: authentication token must be provided', 401)
-
-    auth_scheme = auth_header[:7].title()
-
-    if auth_scheme != 'Bearer ':
-        raise AuthError('Unauthorized: authentication bearer scheme must be used', 401)
-
-    return auth_header[7:]
-
-
-def _get_token_issuer(auth_token):
+def __get_token_issuer(auth_token):
     try:
-        issuer = get_token_issuer(auth_token)
+        client = get_token_issuer(auth_token)
     except TokenIssuerError:
         raise AuthError("Invalid token: iss field not provided", 403)
     except TokenDecodeError:
-        raise AuthError(GENERAL_TOKEN_ERROR_MESSAGE, 403)
-    return issuer
+        raise AuthError("Invalid token: signature, api token is not valid", 403)
+    return client
+
+
+def handle_admin_key(auth_token, secret):
+    try:
+        decode_jwt_token(auth_token, secret)
+    except TokenExpiredError:
+        raise AuthError("Invalid token: expired, check that your system clock is accurate", 403)
+    except TokenDecodeError:
+        raise AuthError("Invalid token: signature, api token is not valid", 403)

app/aws/s3.py

@@ -1,31 +1,28 @@
-import botocore
-from boto3 import Session
+from datetime import datetime, timedelta
+
 from flask import current_app
 
+import pytz
+from boto3 import client, resource
+import botocore
+
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_s3_file(
-    bucket_name, file_location, access_key, secret_key, region
-):
-    s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
+def get_s3_file(bucket_name, file_location):
+    s3_file = get_s3_object(bucket_name, file_location)
     return s3_file.get()['Body'].read().decode('utf-8')
 
 
-def get_s3_object(
-    bucket_name, file_location, access_key, secret_key, region
-):
-    session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
-    s3 = session.resource('s3')
+def get_s3_object(bucket_name, file_location):
+    s3 = resource('s3')
     return s3.Object(bucket_name, file_location)
 
 
-def file_exists(
-    bucket_name, file_location, access_key, secret_key, region
-):
+def file_exists(bucket_name, file_location):
     try:
         # try and access metadata of object
-        get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata
+        get_s3_object(bucket_name, file_location).metadata
         return True
     except botocore.exceptions.ClientError as e:
         if e.response['ResponseMetadata']['HTTPStatusCode'] == 404:
@@ -35,21 +32,8 @@ def file_exists(
 
 def get_job_location(service_id, job_id):
     return (
-        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
+        current_app.config['CSV_UPLOAD_BUCKET_NAME'],
         FILE_LOCATION_STRUCTURE.format(service_id, job_id),
-        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
-        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
-        current_app.config['CSV_UPLOAD_BUCKET']['region'],
-    )
-
-
-def get_contact_list_location(service_id, contact_list_id):
-    return (
-        current_app.config['CONTACT_LIST_BUCKET']['bucket'],
-        FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
-        current_app.config['CONTACT_LIST_BUCKET']['access_key_id'],
-        current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'],
-        current_app.config['CONTACT_LIST_BUCKET']['region'],
     )
 
 
@@ -72,10 +56,59 @@ def remove_job_from_s3(service_id, job_id):
     return remove_s3_object(*get_job_location(service_id, job_id))
 
 
-def remove_contact_list_from_s3(service_id, contact_list_id):
-    return remove_s3_object(*get_contact_list_location(service_id, contact_list_id))
+def get_s3_bucket_objects(bucket_name, subfolder='', older_than=7, limit_days=2):
+    boto_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = boto_client.get_paginator('list_objects_v2')
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    all_objects_in_bucket = []
+    for page in page_iterator:
+        if page.get('Contents'):
+            all_objects_in_bucket.extend(page['Contents'])
+
+    return all_objects_in_bucket
 
 
-def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
-    obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
+def filter_s3_bucket_objects_within_date_range(bucket_objects, older_than=7, limit_days=2):
+    """
+    S3 returns the Object['LastModified'] as an 'offset-aware' timestamp so the
+    date range filter must take this into account.
+
+    Additionally an additional Object is returned by S3 corresponding to the
+    container directory. This is redundant and should be removed.
+
+    """
+    end_date = datetime.now(tz=pytz.utc) - timedelta(days=older_than)
+    start_date = end_date - timedelta(days=limit_days)
+    filtered_items = [item for item in bucket_objects if all([
+        not item['Key'].endswith('/'),
+        item['LastModified'] > start_date,
+        item['LastModified'] < end_date
+    ])]
+
+    return filtered_items
+
+
+def remove_s3_object(bucket_name, object_key):
+    obj = get_s3_object(bucket_name, object_key)
     return obj.delete()
+
+
+def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None):
+    s3_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = s3_client.get_paginator('list_objects_v2')
+
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    for page in page_iterator:
+        for obj in page.get('Contents', []):
+            key = obj['Key']
+            if key.lower().endswith(suffix.lower()):
+                if not last_modified or obj['LastModified'] >= last_modified:
+                    yield key

app/billing/billing_schemas.py

@@ -1,44 +1,42 @@
 from datetime import datetime
 
+
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
     "type": "object",
     "title": "Create",
     "properties": {
-        "free_sms_fragment_limit": {"type": "integer", "minimum": 0},
+        "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
     },
     "required": ["free_sms_fragment_limit"]
 }
 
 
-def serialize_ft_billing_remove_emails(rows):
-    return [
-        {
-            "month": (datetime.strftime(row.month, "%B")),
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_remove_emails(data):
+    results = []
+    billed_notifications = [x for x in data if x.notification_type != 'email']
+    for notification in billed_notifications:
+        json_result = {
+            "month": (datetime.strftime(notification.month, "%B")),
+            "notification_type": notification.notification_type,
+            "billing_units": notification.billable_units,
+            "rate": float(notification.rate),
+            "postage": notification.postage,
         }
-        for row in rows
-        if row.notification_type != 'email'
-    ]
+        results.append(json_result)
+    return results
 
 
-def serialize_ft_billing_yearly_totals(rows):
-    return [
-        {
-            "notification_type": row.notification_type,
-            "chargeable_units": row.chargeable_units,
-            "notifications_sent": row.notifications_sent,
-            "rate": float(row.rate),
-            "cost": float(row.cost),
-            "free_allowance_used": row.free_allowance_used,
-            "charged_units": row.charged_units,
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
         }
-        for row in rows
-    ]
+        yearly_totals.append(json_result)
+
+    return yearly_totals

app/billing/rest.py

@@ -6,18 +6,18 @@ from app.billing.billing_schemas import (
     serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
     dao_get_free_sms_fragment_limit_for_year,
-    dao_update_annual_billing_for_future_years,
-    set_default_free_allowance_for_service,
+    dao_get_all_free_sms_fragment_limit,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
-    fetch_billing_totals_for_year,
-    fetch_monthly_billing_for_year,
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
 )
+
+from app.errors import InvalidRequest
 from app.errors import register_errors
-from app.models import Service
 from app.schema_validation import validate
 
 billing_blueprint = Blueprint(
@@ -30,6 +30,7 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
+@billing_blueprint.route('/ft-monthly-usage')
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
@@ -41,6 +42,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
+@billing_blueprint.route('/ft-yearly-usage-summary')
 @billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:
@@ -61,14 +63,27 @@ def get_free_sms_fragment_limit(service_id):
     annual_billing = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
     if annual_billing is None:
-        service = Service.query.get(service_id)
-        # An entry does not exist in annual_billing table for that service and year.
-        # Set the annual billing to the default free allowance based on the organisation type of the service.
+        # An entry does not exist in annual_billing table for that service and year. If it is a past year,
+        # we return the oldest entry.
+        # If it is the current or future years, we create an entry in the db table using the newest record,
+        # and return that number.  If all fails, we return InvalidRequest.
+        sms_list = dao_get_all_free_sms_fragment_limit(service_id)
 
-        annual_billing = set_default_free_allowance_for_service(
-            service=service,
-            year_start=int(financial_year_start) if financial_year_start else None
-        )
+        if not sms_list:
+            raise InvalidRequest('no free-sms-fragment-limit entry for service {} in DB'.format(service_id), 404)
+        else:
+            if financial_year_start is None:
+                financial_year_start = get_current_financial_year_start_year()
+
+            if int(financial_year_start) < get_current_financial_year_start_year():
+                # return the earliest historical entry
+                annual_billing = sms_list[0]   # The oldest entry
+            else:
+                annual_billing = sms_list[-1]  # The newest entry
+
+                annual_billing = dao_create_or_update_annual_billing_for_year(service_id,
+                                                                              annual_billing.free_sms_fragment_limit,
+                                                                              financial_year_start)
 
     return jsonify(annual_billing.serialize_free_sms_items()), 200
 

app/celery/celery.py

@@ -0,0 +1,62 @@
+import time
+
+from celery import Celery, Task
+from celery.signals import worker_process_shutdown
+from flask import current_app, g, request
+from flask.ctx import has_request_context
+
+
+@worker_process_shutdown.connect
+def worker_process_shutdown(sender, signal, pid, exitcode, **kwargs):
+    current_app.logger.info('worker shutdown: PID: {} Exitcode: {}'.format(pid, exitcode))
+
+
+def make_task(app):
+    class NotifyTask(Task):
+        abstract = True
+        start = None
+
+        def on_success(self, retval, task_id, args, kwargs):
+            elapsed_time = time.time() - self.start
+            app.logger.info(
+                "{task_name} took {time}".format(
+                    task_name=self.name, time="{0:.4f}".format(elapsed_time)
+                )
+            )
+
+        def on_failure(self, exc, task_id, args, kwargs, einfo):
+            # ensure task will log exceptions to correct handlers
+            app.logger.exception('Celery task: {} failed'.format(self.name))
+            super().on_failure(exc, task_id, args, kwargs, einfo)
+
+        def __call__(self, *args, **kwargs):
+            # ensure task has flask context to access config, logger, etc
+            with app.app_context():
+                self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
+                return super().__call__(*args, **kwargs)
+
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
+    return NotifyTask
+
+
+class NotifyCelery(Celery):
+
+    def init_app(self, app):
+        super().__init__(
+            app.import_name,
+            broker=app.config['BROKER_URL'],
+            task_cls=make_task(app),
+        )
+
+        self.conf.update(app.config)

app/celery/letters_pdf_tasks.py

@@ -0,0 +1,525 @@
+import math
+import base64
+from datetime import datetime
+from uuid import UUID
+from hashlib import sha512
+from base64 import urlsafe_b64encode
+
+from botocore.exceptions import ClientError as BotoClientError
+from flask import current_app
+from requests import (
+    post as requests_post,
+    RequestException
+)
+from celery.exceptions import MaxRetriesExceededError
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.s3 import s3upload
+
+from app import encryption, notify_celery
+from app.aws import s3
+from app.config import QueueNames, TaskNames
+from app.dao.notifications_dao import (
+    get_notification_by_id,
+    update_notification_status_by_id,
+    dao_update_notification,
+    dao_get_notification_by_reference,
+    dao_get_notifications_by_references,
+    dao_update_notifications_by_reference,
+)
+from app.errors import VirusScanError
+from app.exceptions import NotificationTechnicalFailureException
+from app.letters.utils import (
+    copy_redaction_failed_pdf,
+    get_billable_units_for_letter_page_count,
+    get_reference_from_filename,
+    get_folder_name,
+    upload_letter_pdf,
+    ScanErrorType,
+    move_failed_pdf,
+    move_sanitised_letter_to_test_or_live_pdf_bucket,
+    move_scan_to_invalid_pdf_bucket,
+    move_error_pdf_to_scan_bucket,
+    get_file_names_from_error_bucket,
+)
+from app.models import (
+    KEY_TYPE_TEST,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_PENDING_VIRUS_CHECK,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_VALIDATION_FAILED,
+    NOTIFICATION_VIRUS_SCAN_FAILED,
+)
+from app.cronitor import cronitor
+
+
+@notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+def create_letters_pdf(self, notification_id):
+    try:
+        notification = get_notification_by_id(notification_id, _raise=True)
+        pdf_data, billable_units = get_letters_pdf(
+            notification.template,
+            contact_block=notification.reply_to_text,
+            filename=notification.service.letter_branding and notification.service.letter_branding.filename,
+            values=notification.personalisation
+        )
+
+        upload_letter_pdf(notification, pdf_data)
+
+        if notification.key_type != KEY_TYPE_TEST:
+            notification.billable_units = billable_units
+            dao_update_notification(notification)
+
+        current_app.logger.info(
+            'Letter notification reference {reference}: billable units set to {billable_units}'.format(
+                reference=str(notification.reference), billable_units=billable_units))
+
+    except (RequestException, BotoClientError):
+        try:
+            current_app.logger.exception(
+                "Letters PDF notification creation for id: {} failed".format(notification_id)
+            )
+            self.retry(queue=QueueNames.RETRY)
+        except MaxRetriesExceededError:
+            current_app.logger.error(
+                "RETRY FAILED: task create_letters_pdf failed for notification {}".format(notification_id),
+            )
+            update_notification_status_by_id(notification_id, 'technical-failure')
+
+
+def get_letters_pdf(template, contact_block, filename, values):
+    template_for_letter_print = {
+        "subject": template.subject,
+        "content": template.content
+    }
+
+    data = {
+        'letter_contact_block': contact_block,
+        'template': template_for_letter_print,
+        'values': values,
+        'filename': filename,
+    }
+    resp = requests_post(
+        '{}/print.pdf'.format(
+            current_app.config['TEMPLATE_PREVIEW_API_HOST']
+        ),
+        json=data,
+        headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+    )
+    resp.raise_for_status()
+
+    pages_per_sheet = 2
+    billable_units = math.ceil(int(resp.headers.get("X-pdf-page-count", 0)) / pages_per_sheet)
+
+    return resp.content, billable_units
+
+
+@notify_celery.task(name='collate-letter-pdfs-for-day')
+@cronitor("collate-letter-pdfs-for-day")
+def collate_letter_pdfs_for_day(date=None):
+    if not date:
+        # Using the truncated date is ok because UTC to BST does not make a difference to the date,
+        # since it is triggered mid afternoon.
+        date = datetime.utcnow().strftime("%Y-%m-%d")
+
+    letter_pdfs = sorted(
+        s3.get_s3_bucket_objects(
+            current_app.config['LETTERS_PDF_BUCKET_NAME'],
+            subfolder=date
+        ),
+        key=lambda letter: letter['Key']
+    )
+    for i, letters in enumerate(group_letters(letter_pdfs)):
+        filenames = [letter['Key'] for letter in letters]
+
+        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+            date=date,
+            num=i + 1,
+            hash=hash
+        )
+
+        current_app.logger.info(
+            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                len(filenames),
+                dvla_filename,
+                sum(letter['Size'] for letter in letters)
+            )
+        )
+        notify_celery.send_task(
+            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+            kwargs={
+                'filenames_to_zip': filenames,
+                'upload_filename': dvla_filename
+            },
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
+        )
+
+
+def group_letters(letter_pdfs):
+    """
+    Group letters in chunks of MAX_LETTER_PDF_ZIP_FILESIZE. Will add files to lists, never going over that size.
+    If a single file is (somehow) larger than MAX_LETTER_PDF_ZIP_FILESIZE that'll be in a list on it's own.
+    If there are no files, will just exit (rather than yielding an empty list).
+    """
+    running_filesize = 0
+    list_of_files = []
+    for letter in letter_pdfs:
+        if letter['Key'].lower().endswith('.pdf') and letter_in_created_state(letter['Key']):
+            if (
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
+            ):
+                yield list_of_files
+                running_filesize = 0
+                list_of_files = []
+
+            running_filesize += letter['Size']
+            list_of_files.append(letter)
+
+    if list_of_files:
+        yield list_of_files
+
+
+def letter_in_created_state(filename):
+    # filename looks like '2018-01-13/NOTIFY.ABCDEF1234567890.D.2.C.C.20180113120000.PDF'
+    subfolder = filename.split('/')[0]
+    ref = get_reference_from_filename(filename)
+    notifications = dao_get_notifications_by_references([ref])
+    if notifications:
+        if notifications[0].status == NOTIFICATION_CREATED:
+            return True
+        current_app.logger.info('Collating letters for {} but notification with reference {} already in {}'.format(
+            subfolder,
+            ref,
+            notifications[0].status
+        ))
+    return False
+
+
+@notify_celery.task(bind=True, name='process-virus-scan-passed', max_retries=15, default_retry_delay=300)
+def process_virus_scan_passed(self, filename):
+    reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
+    current_app.logger.info('notification id {} Virus scan passed: {}'.format(notification.id, filename))
+
+    is_test_key = notification.key_type == KEY_TYPE_TEST
+
+    scan_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
+    old_pdf = scan_pdf_object.get()['Body'].read()
+
+    sanitise_response, result = _sanitise_precompiled_pdf(self, notification, old_pdf)
+    new_pdf = None
+    if result == 'validation_passed':
+        new_pdf = base64.b64decode(sanitise_response["file"].encode())
+
+        redaction_failed_message = sanitise_response.get("redaction_failed_message")
+        if redaction_failed_message and not is_test_key:
+            current_app.logger.info('{} for notification id {} ({})'.format(
+                redaction_failed_message, notification.id, filename)
+            )
+            copy_redaction_failed_pdf(filename)
+
+    billable_units = get_billable_units_for_letter_page_count(sanitise_response.get("page_count"))
+
+    # TODO: Remove this once CYSP update their template to not cross over the margins
+    if notification.service_id == UUID('fe44178f-3b45-4625-9f85-2264a36dd9ec'):  # CYSP
+        # Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
+        new_pdf = old_pdf
+
+    if result == 'validation_failed' and not new_pdf:
+        current_app.logger.info('Invalid precompiled pdf received {} ({})'.format(notification.id, filename))
+        _move_invalid_letter_and_update_status(
+            notification=notification,
+            filename=filename,
+            scan_pdf_object=scan_pdf_object,
+            message=sanitise_response["message"],
+            invalid_pages=sanitise_response.get("invalid_pages"),
+            page_count=sanitise_response.get("page_count")
+        )
+        return
+
+    current_app.logger.info('notification id {} ({}) sanitised and ready to send'.format(notification.id, filename))
+
+    try:
+        _upload_pdf_to_test_or_live_pdf_bucket(
+            new_pdf,
+            filename,
+            is_test_letter=is_test_key,
+            created_at=notification.created_at
+        )
+
+        update_letter_pdf_status(
+            reference=reference,
+            status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
+            billable_units=billable_units,
+            recipient_address=sanitise_response.get("recipient_address")
+        )
+        scan_pdf_object.delete()
+    except BotoClientError:
+        current_app.logger.exception(
+            "Error uploading letter to live pdf bucket for notification: {}".format(notification.id)
+        )
+        update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
+
+
+@notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300)
+def sanitise_letter(self, filename):
+    try:
+        reference = get_reference_from_filename(filename)
+        notification = dao_get_notification_by_reference(reference)
+
+        current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))
+
+        if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
+            current_app.logger.info('Sanitise letter called for notification {} which has is in {} state'.format(
+                notification.id, notification.status))
+            return
+
+        notify_celery.send_task(
+            name=TaskNames.SANITISE_LETTER,
+            kwargs={
+                'notification_id': str(notification.id),
+                'filename': filename,
+            },
+            queue=QueueNames.SANITISE_LETTERS,
+        )
+    except Exception:
+        try:
+            current_app.logger.exception(
+                "RETRY: calling sanitise_letter task for notification {} failed".format(notification.id)
+            )
+            self.retry(queue=QueueNames.RETRY)
+        except self.MaxRetriesExceededError:
+            message = "RETRY FAILED: Max retries reached. " \
+                      "The task sanitise_letter failed for notification {}. " \
+                      "Notification has been updated to technical-failure".format(notification.id)
+            update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
+            raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='process-sanitised-letter')
+def process_sanitised_letter(sanitise_data):
+    letter_details = encryption.decrypt(sanitise_data)
+
+    filename = letter_details['filename']
+    notification_id = letter_details['notification_id']
+
+    current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id))
+    notification = get_notification_by_id(notification_id, _raise=True)
+
+    if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
+        current_app.logger.info(
+            'process-sanitised-letter task called for notification {} which has is in {} state'.format(
+                notification.id, notification.status)
+        )
+        return
+
+    try:
+        original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
+
+        if letter_details['validation_status'] == 'failed':
+            current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format(
+                notification_id, filename))
+
+            _move_invalid_letter_and_update_status(
+                notification=notification,
+                filename=filename,
+                scan_pdf_object=original_pdf_object,
+                message=letter_details['message'],
+                invalid_pages=letter_details['invalid_pages'],
+                page_count=letter_details['page_count'],
+            )
+            return
+
+        current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format(
+            notification_id, filename))
+
+        billable_units = get_billable_units_for_letter_page_count(letter_details['page_count'])
+        is_test_key = notification.key_type == KEY_TYPE_TEST
+
+        move_sanitised_letter_to_test_or_live_pdf_bucket(filename, is_test_key, notification.created_at)
+        # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
+        original_pdf_object.delete()
+        update_letter_pdf_status(
+            reference=notification.reference,
+            status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
+            billable_units=billable_units,
+            recipient_address=letter_details['address']
+        )
+
+    except BotoClientError:
+        current_app.logger.exception(
+            "Boto error when processing sanitised letter for notification {}".format(filename, notification.id)
+        )
+        update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
+        raise NotificationTechnicalFailureException
+
+
+def _move_invalid_letter_and_update_status(
+    *, notification, filename, scan_pdf_object, message=None, invalid_pages=None, page_count=None
+):
+    try:
+        move_scan_to_invalid_pdf_bucket(
+            source_filename=filename,
+            message=message,
+            invalid_pages=invalid_pages,
+            page_count=page_count
+        )
+        scan_pdf_object.delete()
+
+        update_letter_pdf_status(
+            reference=notification.reference,
+            status=NOTIFICATION_VALIDATION_FAILED,
+            billable_units=0)
+    except BotoClientError:
+        current_app.logger.exception(
+            "Error when moving letter with id {} to invalid PDF bucket".format(notification.id)
+        )
+        update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
+        raise NotificationTechnicalFailureException
+
+
+def _upload_pdf_to_test_or_live_pdf_bucket(pdf_data, filename, is_test_letter, created_at):
+    target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME'
+    target_bucket_name = current_app.config[target_bucket_config]
+    target_filename = get_folder_name(created_at, dont_use_sending_date=is_test_letter) + filename
+
+    s3upload(
+        filedata=pdf_data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=target_bucket_name,
+        file_location=target_filename
+    )
+
+
+def _sanitise_precompiled_pdf(self, notification, precompiled_pdf):
+    try:
+        response = requests_post(
+            '{}/precompiled/sanitise'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST']
+            ),
+            data=precompiled_pdf,
+            headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY']),
+                     'Service-ID': str(notification.service_id),
+                     'Notification-ID': str(notification.id)}
+        )
+        response.raise_for_status()
+        return response.json(), "validation_passed"
+    except RequestException as ex:
+        if ex.response is not None and ex.response.status_code == 400:
+            message = "sanitise_precompiled_pdf validation error for notification: {}. ".format(notification.id)
+            if response.json().get("message"):
+                message += response.json()["message"]
+                if response.json().get("invalid_pages"):
+                    message += (" on pages: " + ", ".join(map(str, response.json()["invalid_pages"])))
+
+            current_app.logger.info(
+                message
+            )
+            return response.json(), "validation_failed"
+
+        try:
+            current_app.logger.exception(
+                "sanitise_precompiled_pdf failed for notification: {}".format(notification.id)
+            )
+            self.retry(queue=QueueNames.RETRY)
+        except MaxRetriesExceededError:
+            current_app.logger.error(
+                "RETRY FAILED: sanitise_precompiled_pdf failed for notification {}".format(notification.id),
+            )
+
+            notification.status = NOTIFICATION_TECHNICAL_FAILURE
+            dao_update_notification(notification)
+            raise
+
+
+@notify_celery.task(name='process-virus-scan-failed')
+def process_virus_scan_failed(filename):
+    move_failed_pdf(filename, ScanErrorType.FAILURE)
+    reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
+    updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0)
+
+    if updated_count != 1:
+        raise Exception(
+            "There should only be one letter notification for each reference. Found {} notifications".format(
+                updated_count
+            )
+        )
+
+    error = VirusScanError('notification id {} Virus scan failed: {}'.format(notification.id, filename))
+    current_app.logger.exception(error)
+    raise error
+
+
+@notify_celery.task(name='process-virus-scan-error')
+def process_virus_scan_error(filename):
+    move_failed_pdf(filename, ScanErrorType.ERROR)
+    reference = get_reference_from_filename(filename)
+    notification = dao_get_notification_by_reference(reference)
+    updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE, billable_units=0)
+
+    if updated_count != 1:
+        raise Exception(
+            "There should only be one letter notification for each reference. Found {} notifications".format(
+                updated_count
+            )
+        )
+    error = VirusScanError('notification id {} Virus scan error: {}'.format(notification.id, filename))
+    current_app.logger.exception(error)
+    raise error
+
+
+def update_letter_pdf_status(reference, status, billable_units, recipient_address=None):
+
+    update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()}
+    if recipient_address:
+        update_dict['to'] = recipient_address
+    return dao_update_notifications_by_reference(
+        references=[reference],
+        update_dict=update_dict)[0]
+
+
+def replay_letters_in_error(filename=None):
+    # This method can be used to replay letters that end up in the ERROR directory.
+    # We had an incident where clamAV was not processing the virus scan.
+    if filename:
+        move_error_pdf_to_scan_bucket(filename)
+        # call task to add the filename to anti virus queue
+        current_app.logger.info("Calling scan_file for: {}".format(filename))
+
+        if current_app.config['ANTIVIRUS_ENABLED']:
+            notify_celery.send_task(
+                name=TaskNames.SCAN_FILE,
+                kwargs={'filename': filename},
+                queue=QueueNames.ANTIVIRUS,
+            )
+        else:
+            # stub out antivirus in dev
+            process_virus_scan_passed.apply_async(
+                kwargs={'filename': filename},
+                queue=QueueNames.LETTERS,
+            )
+    else:
+        error_files = get_file_names_from_error_bucket()
+        for item in error_files:
+            moved_file_name = item.key.split('/')[1]
+            current_app.logger.info("Calling scan_file for: {}".format(moved_file_name))
+            move_error_pdf_to_scan_bucket(moved_file_name)
+            # call task to add the filename to anti virus queue
+            if current_app.config['ANTIVIRUS_ENABLED']:
+                notify_celery.send_task(
+                    name=TaskNames.SCAN_FILE,
+                    kwargs={'filename': moved_file_name},
+                    queue=QueueNames.ANTIVIRUS,
+                )
+            else:
+                # stub out antivirus in dev
+                process_virus_scan_passed.apply_async(
+                    kwargs={'filename': moved_file_name},
+                    queue=QueueNames.LETTERS,
+                )

app/celery/nightly_tasks.py

@@ -1,39 +1,59 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
+import pytz
 from flask import current_app
-from notifications_utils.timezones import convert_utc_to_local_timezone
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, statsd_client
+from app import notify_celery, performance_platform_client, zendesk_client
 from app.aws import s3
-from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
+from app.celery.service_callback_tasks import (
+    send_delivery_status_to_service,
+    create_delivery_status_callback_data,
+)
 from app.config import QueueNames
-from app.cronitor import cronitor
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
-    dao_archive_job,
     dao_get_jobs_older_than_data_retention,
+    dao_archive_job
 )
 from app.dao.notifications_dao import (
-    dao_get_notifications_processing_time_stats,
     dao_timeout_notifications,
-    get_service_ids_with_notifications_before,
-    move_notifications_to_notification_history,
+    delete_notifications_older_than_retention_by_type,
 )
-from app.dao.service_data_retention_dao import (
-    fetch_service_data_retention_for_all_services_by_notification_type,
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.exceptions import NotificationTechnicalFailureException
+from app.models import (
+    Notification,
+    NOTIFICATION_SENDING,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    LETTER_TYPE,
+    KEY_TYPE_NORMAL
 )
-from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
-from app.utils import get_local_midnight_in_utc
+from app.performance_platform import total_sent_notifications, processing_time
+from app.cronitor import cronitor
+from app.utils import get_london_midnight_in_utc
 
 
 @notify_celery.task(name="remove_sms_email_jobs")
 @cronitor("remove_sms_email_jobs")
+@statsd(namespace="tasks")
 def remove_sms_email_csv_files():
     _remove_csv_files([EMAIL_TYPE, SMS_TYPE])
 
 
+@notify_celery.task(name="remove_letter_jobs")
+@cronitor("remove_letter_jobs")
+@statsd(namespace="tasks")
+def remove_letter_csv_files():
+    _remove_csv_files([LETTER_TYPE])
+
+
 def _remove_csv_files(job_types):
     jobs = dao_get_jobs_older_than_data_retention(notification_types=job_types)
     for job in jobs:
@@ -42,107 +62,142 @@ def _remove_csv_files(job_types):
         current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))
 
 
-@notify_celery.task(name="delete-notifications-older-than-retention")
-def delete_notifications_older_than_retention():
-    delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-    delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
-
-
 @notify_celery.task(name="delete-sms-notifications")
 @cronitor("delete-sms-notifications")
+@statsd(namespace="tasks")
 def delete_sms_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('sms')
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('sms')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} sms notifications".format(
+                'sms',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete sms notifications")
+        raise
 
 
 @notify_celery.task(name="delete-email-notifications")
 @cronitor("delete-email-notifications")
+@statsd(namespace="tasks")
 def delete_email_notifications_older_than_retention():
-    _delete_notifications_older_than_retention_by_type('email')
-
-
-def _delete_notifications_older_than_retention_by_type(notification_type):
-    flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type)
-
-    for f in flexible_data_retention:
-        day_to_delete_backwards_from = get_local_midnight_in_utc(
-            convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
-        )
-
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': f.service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': day_to_delete_backwards_from
-        })
-
-    seven_days_ago = get_local_midnight_in_utc(
-        convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=7)
-    )
-    service_ids_with_data_retention = {x.service_id for x in flexible_data_retention}
-
-    # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services.
-    # This query takes a couple of mins to run.
-    service_ids_that_have_sent_notifications_recently = get_service_ids_with_notifications_before(
-        notification_type,
-        seven_days_ago
-    )
-
-    service_ids_to_purge = service_ids_that_have_sent_notifications_recently - service_ids_with_data_retention
-
-    for service_id in service_ids_to_purge:
-        delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={
-            'service_id': service_id,
-            'notification_type': notification_type,
-            'datetime_to_delete_before': seven_days_ago
-        })
-
-    current_app.logger.info(
-        f'delete-notifications-older-than-retention: triggered subtasks for notification_type {notification_type}: '
-        f'{len(service_ids_with_data_retention)} services with flexible data retention, '
-        f'{len(service_ids_to_purge)} services without flexible data retention'
-    )
-
-
-@notify_celery.task(name='delete-notifications-for-service-and-type')
-def delete_notifications_for_service_and_type(service_id, notification_type, datetime_to_delete_before):
-    start = datetime.utcnow()
-    num_deleted = move_notifications_to_notification_history(
-        notification_type,
-        service_id,
-        datetime_to_delete_before,
-    )
-    if num_deleted:
-        end = datetime.utcnow()
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('email')
         current_app.logger.info(
-            f'delete-notifications-for-service-and-type: '
-            f'service: {service_id}, '
-            f'notification_type: {notification_type}, '
-            f'count deleted: {num_deleted}, '
-            f'duration: {(end - start).seconds} seconds'
+            "Delete {} job started {} finished {} deleted {} email notifications".format(
+                'email',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
         )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete email notifications")
+        raise
+
+
+@notify_celery.task(name="delete-letter-notifications")
+@cronitor("delete-letter-notifications")
+@statsd(namespace="tasks")
+def delete_letter_notifications_older_than_retention():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_notifications_older_than_retention_by_type('letter')
+        current_app.logger.info(
+            "Delete {} job started {} finished {} deleted {} letter notifications".format(
+                'letter',
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete letter notifications")
+        raise
 
 
 @notify_celery.task(name='timeout-sending-notifications')
 @cronitor('timeout-sending-notifications')
+@statsd(namespace="tasks")
 def timeout_notifications():
-    notifications = ['dummy value so len() > 0']
+    technical_failure_notifications, temporary_failure_notifications = \
+        dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
 
-    cutoff_time = datetime.utcnow() - timedelta(
-        seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD')
+    notifications = technical_failure_notifications + temporary_failure_notifications
+    for notification in notifications:
+        # queue callback task only if the service_callback_api exists
+        service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+        if service_callback_api:
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
+            send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+                                                        queue=QueueNames.CALLBACKS)
+
+    current_app.logger.info(
+        "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    if technical_failure_notifications:
+        message = "{} notifications have been updated to technical-failure because they " \
+                  "have timed out and are still in created.Notification ids: {}".format(
+                      len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
+        raise NotificationTechnicalFailureException(message)
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@cronitor('send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_platform_stats(date=None):
+    # date is a string in the format of "YYYY-MM-DD"
+    if date is None:
+        date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        date = datetime.strptime(date, "%Y-%m-%d").date()
+
+    if performance_platform_client.active:
+
+        send_total_sent_notifications_to_performance_platform(bst_date=date)
+        processing_time.send_processing_time_to_performance_platform(bst_date=date)
+
+
+def send_total_sent_notifications_to_performance_platform(bst_date):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(bst_date)
+    start_time = get_london_midnight_in_utc(bst_date)
+
+    email_sent_count = count_dict['email']
+    sms_sent_count = count_dict['sms']
+    letter_sent_count = count_dict['letter']
+
+    current_app.logger.info(
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(bst_date, email_sent_count, sms_sent_count, letter_sent_count)
     )
 
-    while len(notifications) > 0:
-        notifications = dao_timeout_notifications(cutoff_time)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'sms',
+        sms_sent_count
+    )
 
-        for notification in notifications:
-            statsd_client.incr(f'timeout-sending.{notification.sent_by}')
-            check_and_queue_callback_task(notification)
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'email',
+        email_sent_count
+    )
 
-        current_app.logger.info(
-            "Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_time,
+        'letter',
+        letter_sent_count
+    )
 
 
 @notify_celery.task(name="delete-inbound-sms")
 @cronitor("delete-inbound-sms")
+@statsd(namespace="tasks")
 def delete_inbound_sms():
     try:
         start = datetime.utcnow()
@@ -159,23 +214,125 @@ def delete_inbound_sms():
         raise
 
 
-@notify_celery.task(name='save-daily-notification-processing-time')
-@cronitor("save-daily-notification-processing-time")
-def save_daily_notification_processing_time(local_date=None):
-    # local_date is a string in the format of "YYYY-MM-DD"
-    if local_date is None:
-        # if a date is not provided, we run against yesterdays data
-        local_date = (datetime.utcnow() - timedelta(days=1)).date()
-    else:
-        local_date = datetime.strptime(local_date, "%Y-%m-%d").date()
-
-    start_time = get_local_midnight_in_utc(local_date)
-    end_time = get_local_midnight_in_utc(local_date + timedelta(days=1))
-    result = dao_get_notifications_processing_time_stats(start_time, end_time)
-    insert_update_processing_time(
-        FactProcessingTime(
-            local_date=local_date,
-            messages_total=result.messages_total,
-            messages_within_10_secs=result.messages_within_10_secs
+# TODO: remove me, i'm not being run by anything
+@notify_celery.task(name="delete_dvla_response_files")
+@statsd(namespace="tasks")
+def delete_dvla_response_files_older_than_seven_days():
+    try:
+        start = datetime.utcnow()
+        bucket_objects = s3.get_s3_bucket_objects(
+            current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+            'root/dispatch'
         )
+        older_than_seven_days = s3.filter_s3_bucket_objects_within_date_range(bucket_objects)
+
+        for f in older_than_seven_days:
+            s3.remove_s3_object(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], f['Key'])
+
+        current_app.logger.info(
+            "Delete dvla response files started {} finished {} deleted {} files".format(
+                start,
+                datetime.utcnow(),
+                len(older_than_seven_days)
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to delete dvla response files")
+        raise
+
+
+@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
+@cronitor("raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
+def raise_alert_if_letter_notifications_still_sending():
+    today = datetime.utcnow().date()
+
+    # Do nothing on the weekend
+    if today.isoweekday() in {6, 7}:  # sat, sun
+        return
+
+    if today.isoweekday() in {1, 2}:  # mon, tues. look for files from before the weekend
+        offset_days = 4
+    else:
+        offset_days = 2
+
+    q = Notification.query.filter(
+        Notification.notification_type == LETTER_TYPE,
+        Notification.status == NOTIFICATION_SENDING,
+        Notification.key_type == KEY_TYPE_NORMAL,
+        func.date(Notification.sent_at) <= today - timedelta(days=offset_days)
     )
+
+    if today.isoweekday() in {2, 4}:  # on tue, thu, we only care about first class letters
+        q = q.filter(
+            Notification.postage == 'first'
+        )
+
+    still_sending = q.count()
+
+    if still_sending:
+        message = "There are {} letters in the 'sending' state from {}".format(
+            still_sending,
+            (today - timedelta(days=offset_days)).strftime('%A %d %B')
+        )
+        # Only send alerts in production
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still sending".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=message,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+        else:
+            current_app.logger.info(message)
+
+
+@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
+@cronitor('raise-alert-if-no-letter-ack-file')
+@statsd(namespace="tasks")
+def letter_raise_alert_if_no_ack_file_for_zip():
+    # get a list of zip files since yesterday
+    zip_file_set = set()
+    today_str = datetime.utcnow().strftime('%Y-%m-%d')
+    yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1)  # AWS datetime format
+
+    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+                                              subfolder=today_str + '/zips_sent',
+                                              suffix='.TXT'):
+        subname = key.split('/')[-1]  # strip subfolder in name
+        zip_file_set.add(subname.upper().replace('.ZIP.TXT', ''))
+
+    # get acknowledgement file
+    ack_file_set = set()
+
+    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+                                              subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
+        ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', ''))
+
+    message = (
+        "Letter ack file does not contain all zip files sent. "
+        "Missing ack for zip files: {}, "
+        "pdf bucket: {}, subfolder: {}, "
+        "ack bucket: {}"
+    ).format(
+        str(sorted(zip_file_set - ack_file_set)),
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
+        current_app.config['DVLA_RESPONSE_BUCKET_NAME']
+    )
+    # strip empty element before comparison
+    ack_file_set.discard('')
+    zip_file_set.discard('')
+
+    if len(zip_file_set - ack_file_set) > 0:
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="Letter acknowledge error",
+                message=message,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+        current_app.logger.error(message)
+
+    if len(ack_file_set - zip_file_set) > 0:
+        current_app.logger.info(
+            "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
+        )

app/celery/process_ses_receipts_tasks.py

@@ -3,101 +3,76 @@ from datetime import datetime, timedelta
 import iso8601
 from celery.exceptions import Retry
 from flask import current_app, json
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
-from app.celery.service_callback_tasks import (
-    create_complaint_callback_data,
-    create_delivery_status_callback_data,
-    send_complaint_to_service,
-    send_delivery_status_to_service,
-)
 from app.config import QueueNames
+from app.clients.email.aws_ses import get_aws_responses
 from app.dao import notifications_dao
-from app.dao.complaint_dao import save_complaint
-from app.dao.notifications_dao import dao_get_notification_history_by_reference
-from app.dao.service_callback_api_dao import (
-    get_service_complaint_callback_api_for_service,
-    get_service_delivery_status_callback_api_for_service,
+from app.models import NOTIFICATION_SENDING, NOTIFICATION_PENDING
+
+from app.notifications.notifications_ses_callback import (
+    determine_notification_bounce_type,
+    handle_complaint,
+    _check_and_queue_complaint_callback_task,
+    _check_and_queue_callback_task,
 )
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def process_ses_results(self, response):
     try:
-        ses_message = json.loads(response["Message"])
-        notification_type = ses_message["notificationType"]
-        # TODO remove after smoke testing on prod is implemented
-        current_app.logger.info(
-            f"Attempting to process SES delivery status message "
-            f"from SNS with type: {notification_type} and body: {ses_message}"
-        )
-        bounce_message = None
+        ses_message = json.loads(response['Message'])
+        notification_type = ses_message['notificationType']
 
         if notification_type == 'Bounce':
-            bounce_message = determine_notification_bounce_type(ses_message)
+            notification_type = determine_notification_bounce_type(notification_type, ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
 
-        aws_response_dict = get_aws_responses(ses_message)
+        aws_response_dict = get_aws_responses(notification_type)
 
-        notification_status = aws_response_dict["notification_status"]
-        reference = ses_message["mail"]["messageId"]
+        notification_status = aws_response_dict['notification_status']
+        reference = ses_message['mail']['messageId']
 
         try:
             notification = notifications_dao.dao_get_notification_by_reference(reference)
         except NoResultFound:
-            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
+            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
-                current_app.logger.info(
-                    f"Notification not found for reference: {reference}"
-                    f"(while attempting update to {notification_status}). "
-                    f"Callback may have arrived before notification was"
-                    f"persisted to the DB. Adding task to retry queue"
-                )
                 self.retry(queue=QueueNames.RETRY)
             else:
                 current_app.logger.warning(
-                    f"Notification not found for reference: {reference} "
-                    f"(while attempting update to {notification_status})"
+                    "notification not found for reference: {} (update to {})".format(reference, notification_status)
                 )
             return
 
-        if bounce_message:
-            current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
-
         if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
-            notifications_dao._duplicate_update_warning(
-                notification,
-                notification_status
-            )
+            notifications_dao._duplicate_update_warning(notification, notification_status)
             return
 
-        notifications_dao._update_notification_status(
-            notification=notification,
-            status=notification_status,
-            provider_response=aws_response_dict["provider_response"],
-        )
+        notifications_dao._update_notification_status(notification=notification, status=notification_status)
 
-        if not aws_response_dict["success"]:
+        if not aws_response_dict['success']:
             current_app.logger.info(
                 "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
-                    notification.id, reference, aws_response_dict["message"]
+                    notification.id, reference, aws_response_dict['message']
                 )
             )
         else:
-            current_app.logger.info(
-                "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
-            )
+            current_app.logger.info('SES callback return status of {} for notification: {}'.format(
+                notification_status, notification.id
+            ))
 
-        statsd_client.incr("callback.ses.{}".format(notification_status))
+        statsd_client.incr('callback.ses.{}'.format(notification_status))
 
         if notification.sent_at:
-            statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at)
+            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)
 
-        check_and_queue_callback_task(notification)
+        _check_and_queue_callback_task(notification)
 
         return True
 
@@ -105,139 +80,5 @@ def process_ses_results(self, response):
         raise
 
     except Exception as e:
-        current_app.logger.exception("Error processing SES results: {}".format(type(e)))
+        current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
         self.retry(queue=QueueNames.RETRY)
-
-
-def determine_notification_bounce_type(ses_message):
-    notification_type = ses_message["notificationType"]
-    if notification_type in ["Delivery", "Complaint"]:
-        return notification_type
-
-    if notification_type != "Bounce":
-        raise KeyError(f"Unhandled sns notification type {notification_type}")
-
-    remove_emails_from_bounce(ses_message)
-    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
-    if ses_message["bounce"]["bounceType"] == "Permanent":
-        return "Permanent"
-    return "Temporary"
-
-
-def determine_notification_type(ses_message):
-    notification_type = ses_message["notificationType"]
-    if notification_type not in ["Bounce", "Complaint",   "Delivery"]:
-        raise KeyError(f"Unhandled sns notification type {notification_type}")
-    if notification_type == 'Bounce':
-        return determine_notification_bounce_type(ses_message)
-    return notification_type
-
-
-def _determine_provider_response(ses_message):
-    if ses_message["notificationType"] != "Bounce":
-        return None
-
-    bounce_type = ses_message["bounce"]["bounceType"]
-    bounce_subtype = ses_message["bounce"]["bounceSubType"]
-
-    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
-    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
-        return "The email address is on our email provider suppression list"
-    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
-        return "The email address is on the GC Notify suppression list"
-    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
-        return "The email was rejected because of its attachments"
-
-    return None
-
-
-def get_aws_responses(ses_message):
-    status = determine_notification_type(ses_message)
-
-    base = {
-        "Permanent": {
-            "message": "Hard bounced",
-            "success": False,
-            "notification_status": "permanent-failure",
-        },
-        "Temporary": {
-            "message": "Soft bounced",
-            "success": False,
-            "notification_status": "temporary-failure",
-        },
-        "Delivery": {
-            "message": "Delivered",
-            "success": True,
-            "notification_status": "delivered",
-        },
-        "Complaint": {
-            "message": "Complaint",
-            "success": True,
-            "notification_status": "delivered",
-        },
-    }[status]
-
-    base["provider_response"] = _determine_provider_response(ses_message)
-
-    return base
-
-
-def handle_complaint(ses_message):
-    recipient_email = remove_emails_from_complaint(ses_message)[0]
-    current_app.logger.info(
-        "Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))
-    )
-    try:
-        reference = ses_message["mail"]["messageId"]
-    except KeyError as e:
-        current_app.logger.exception(f"Complaint from SES failed to get reference from message with error: {e}")
-        return
-    notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get("complaint", None)
-
-    complaint = Complaint(
-        notification_id=notification.id,
-        service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
-        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
-        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
-    )
-    save_complaint(complaint)
-    return complaint, notification, recipient_email
-
-
-def remove_mail_headers(dict_to_edit):
-    if dict_to_edit["mail"].get("headers"):
-        dict_to_edit["mail"].pop("headers")
-    if dict_to_edit["mail"].get("commonHeaders"):
-        dict_to_edit["mail"].pop("commonHeaders")
-
-
-def remove_emails_from_bounce(bounce_dict):
-    remove_mail_headers(bounce_dict)
-    bounce_dict["mail"].pop("destination", None)
-    bounce_dict["bounce"].pop("bouncedRecipients", None)
-
-
-def remove_emails_from_complaint(complaint_dict):
-    remove_mail_headers(complaint_dict)
-    complaint_dict["complaint"].pop("complainedRecipients")
-    return complaint_dict["mail"].pop("destination")
-
-
-def check_and_queue_callback_task(notification):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async(
-            [str(notification.id), notification_data], queue=QueueNames.CALLBACKS
-        )
-
-
-def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
-        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)

app/celery/provider_tasks.py

@@ -1,10 +1,9 @@
 from flask import current_app
+from notifications_utils.recipients import InvalidEmailError
+from notifications_utils.statsd_decorators import statsd
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
-from app.clients.email import EmailClientNonRetryableException
-from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
-from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
@@ -14,6 +13,7 @@ from app.models import NOTIFICATION_TECHNICAL_FAILURE
 
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_sms(self, notification_id):
     try:
         current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id))
@@ -21,18 +21,11 @@ def deliver_sms(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_sms_to_provider(notification)
-    except Exception as e:
-        if isinstance(e, SmsClientResponseException):
-            current_app.logger.warning(
-                "SMS notification delivery for id: {} failed".format(notification_id),
-                exc_info=True
-            )
-        else:
+    except Exception:
+        try:
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
-
-        try:
             if self.request.retries == 0:
                 self.retry(queue=QueueNames.RETRY, countdown=0)
             else:
@@ -45,6 +38,7 @@ def deliver_sms(self, notification_id):
 
 
 @notify_celery.task(bind=True, name="deliver_email", max_retries=48, default_retry_delay=300)
+@statsd(namespace="tasks")
 def deliver_email(self, notification_id):
     try:
         current_app.logger.info("Start sending email for notification id: {}".format(notification_id))
@@ -52,22 +46,14 @@ def deliver_email(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_email_to_provider(notification)
-    except EmailClientNonRetryableException as e:
-        current_app.logger.exception(
-            f"Email notification {notification_id} failed: {e}"
-        )
+    except InvalidEmailError as e:
+        current_app.logger.exception(e)
         update_notification_status_by_id(notification_id, 'technical-failure')
-    except Exception as e:
+    except Exception:
         try:
-            if isinstance(e, AwsSesClientThrottlingSendRateException):
-                current_app.logger.warning(
-                    f"RETRY: Email notification {notification_id} was rate limited by SES"
-                )
-            else:
-                current_app.logger.exception(
-                    f"RETRY: Email notification {notification_id} failed"
-                )
-
+            current_app.logger.exception(
+                "RETRY: Email notification {} failed".format(notification_id)
+            )
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
             message = "RETRY FAILED: Max retries reached. " \

app/celery/reporting_tasks.py

@@ -1,125 +1,102 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.timezones import convert_utc_to_local_timezone
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.timezones import convert_utc_to_bst
 
 from app import notify_celery
 from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_billing_dao import (
     fetch_billing_data_for_day,
-    update_fact_billing,
+    update_fact_billing
+)
+from app.dao.fact_notification_status_dao import fetch_notification_status_for_day, update_fact_notification_status
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    LETTER_TYPE,
 )
-from app.dao.fact_notification_status_dao import update_fact_notification_status
-from app.dao.notifications_dao import get_service_ids_with_notifications_on_date
-from app.models import EMAIL_TYPE, SMS_TYPE
 
 
 @notify_celery.task(name="create-nightly-billing")
 @cronitor("create-nightly-billing")
+@statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
     # day_start is a datetime.date() object. e.g.
     # up to 4 days of data counting back from day_start is consolidated
     if day_start is None:
-        day_start = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
+        day_start = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
     else:
         # When calling the task its a string in the format of "YYYY-MM-DD"
         day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 10):
-        process_day = (day_start - timedelta(days=i)).isoformat()
+    for i in range(0, 4):
+        process_day = day_start - timedelta(days=i)
 
         create_nightly_billing_for_day.apply_async(
-            kwargs={'process_day': process_day},
+            kwargs={'process_day': process_day.isoformat()},
             queue=QueueNames.REPORTING
         )
-        current_app.logger.info(
-            f"create-nightly-billing task: create-nightly-billing-for-day task created for {process_day}"
-        )
 
 
 @notify_celery.task(name="create-nightly-billing-for-day")
+@statsd(namespace="tasks")
 def create_nightly_billing_for_day(process_day):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
-    current_app.logger.info(
-        f'create-nightly-billing-for-day task for {process_day}: started'
-    )
 
     start = datetime.utcnow()
     transit_data = fetch_billing_data_for_day(process_day=process_day)
     end = datetime.utcnow()
 
-    current_app.logger.info(
-        f'create-nightly-billing-for-day task for {process_day}: data fetched in {(end - start).seconds} seconds'
-    )
+    current_app.logger.info(f'create-nightly-billing-for-day {process_day} fetched in {(end - start).seconds} seconds')
 
     for data in transit_data:
         update_fact_billing(data, process_day)
 
     current_app.logger.info(
-        f"create-nightly-billing-for-day task for {process_day}: "
-        f"task complete. {len(transit_data)} rows updated"
+        f"create-nightly-billing-for-day task complete. {len(transit_data)} rows updated for day: {process_day}"
     )
 
 
 @notify_celery.task(name="create-nightly-notification-status")
 @cronitor("create-nightly-notification-status")
+@statsd(namespace="tasks")
 def create_nightly_notification_status():
-    """
-    Aggregate notification statuses into rows in ft_notification_status.
-    In order to minimise effort, this task assumes that:
+    yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
 
-      - Email + SMS statuses don't change after 3 days. This is currently true
-        because all outstanding email / SMS are "timed out" after 3 days, and
-        we reject delivery receipts after this point.
-
-    Because the time range of the task exceeds the minimum possible retention
-    period (3 days), we need to choose which table to query for each service.
-
-    The aggregation happens for 1 extra day in case:
-
-      - This task or the "timeout" task fails to run.
-
-      - Data is (somehow) still in transit to the history table, which would
-        mean the aggregated results are temporarily incorrect.
-    """
-
-    yesterday = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1)
-
-    for notification_type in [SMS_TYPE, EMAIL_TYPE]:
-        days = 4
-
-        for i in range(days):
-            process_day = yesterday - timedelta(days=i)
-
-            relevant_service_ids = get_service_ids_with_notifications_on_date(
-                notification_type, process_day
+    # email and sms
+    for i in range(4):
+        process_day = yesterday - timedelta(days=i)
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
             )
-
-            for service_id in relevant_service_ids:
-                create_nightly_notification_status_for_service_and_day.apply_async(
-                    kwargs={
-                        'process_day': process_day.isoformat(),
-                        'notification_type': notification_type,
-                        'service_id': service_id,
-                    },
-                    queue=QueueNames.REPORTING
-                )
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
+    for i in range(10):
+        process_day = yesterday - timedelta(days=i)
+        create_nightly_notification_status_for_day.apply_async(
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )
 
 
-@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
-def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
+@notify_celery.task(name="create-nightly-notification-status-for-day")
+@statsd(namespace="tasks")
+def create_nightly_notification_status_for_day(process_day, notification_type):
     process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
 
     start = datetime.utcnow()
-    update_fact_notification_status(
-        process_day=process_day,
-        notification_type=notification_type,
-        service_id=service_id
-    )
-
+    transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
     end = datetime.utcnow()
     current_app.logger.info(
-        f'create-nightly-notification-status-for-service-and-day task update '
-        f'for {service_id}, {notification_type} for {process_day}: '
-        f'updated in {(end - start).seconds} seconds'
+        f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
+        f'data fetched in {(end - start).seconds} seconds'
+    )
+
+    update_fact_notification_status(transit_data, process_day, notification_type)
+
+    current_app.logger.info(
+        f'create-nightly-notification-status-for-day {process_day} type {notification_type} task: '
+        f'task complete - {len(transit_data)} rows updated'
     )

app/celery/research_mode_tasks.py

@@ -1,15 +1,21 @@
+import random
+from datetime import datetime, timedelta
 import json
 
 from flask import current_app
-from requests import HTTPError, request
+from requests import request, RequestException, HTTPError
 
-from app.celery.process_ses_receipts_tasks import process_ses_results
-from app.config import QueueNames
+from notifications_utils.s3 import s3upload
+
+from app import notify_celery
+from app.aws.s3 import file_exists
 from app.models import SMS_TYPE
+from app.config import QueueNames
+from app.celery.process_ses_receipts_tasks import process_ses_results
 
-temp_fail = "2028675303"
-perm_fail = "2028675302"
-delivered = "2028675309"
+temp_fail = "7700900003"
+perm_fail = "7700900002"
+delivered = "7700900001"
 
 delivered_email = "delivered@simulator.notify"
 perm_fail_email = "perm-fail@simulator.notify"
@@ -17,8 +23,22 @@ temp_fail_email = "temp-fail@simulator.notify"
 
 
 def send_sms_response(provider, reference, to):
-    body = sns_callback(reference, to)
-    headers = {"Content-type": "application/json"}
+    if provider == "mmg":
+        body = mmg_callback(reference, to)
+        headers = {"Content-type": "application/json"}
+    else:
+        headers = {"Content-type": "application/x-www-form-urlencoded"}
+        body = firetext_callback(reference, to)
+        # to simulate getting a temporary_failure from firetext
+        # we need to send a pending status updated then a permanent-failure
+        if body['status'] == '2':  # pending status
+            make_request(SMS_TYPE, provider, body, headers)
+            # 1 is a declined status for firetext, will result in a temp-failure
+            body = {'mobile': to,
+                    'status': "1",
+                    'time': '2016-03-10 14:17:00',
+                    'reference': reference
+                    }
 
     make_request(SMS_TYPE, provider, body, headers)
 
@@ -46,38 +66,99 @@ def make_request(notification_type, provider, data, headers):
             timeout=60
         )
         response.raise_for_status()
-    except HTTPError as e:
+    except RequestException as e:
+        api_error = HTTPError(e)
         current_app.logger.error(
-            "API POST request on {} failed with status {}".format(
+            "API {} request on {} failed with {}".format(
+                "POST",
                 api_call,
-                e.response.status_code
+                api_error.response
             )
         )
-        raise e
+        raise api_error
     finally:
         current_app.logger.info("Mocked provider callback request finished")
     return response.json()
 
 
-def sns_callback(notification_id, to):
-    raise Exception("Need to update for SNS callback format along with test_send_to_providers")
+def mmg_callback(notification_id, to):
+    """
+        status: 3 - delivered
+        status: 4 - expired (temp failure)
+        status: 5 - rejected (perm failure)
+    """
 
-    # example from mmg_callback
-    # if to.strip().endswith(temp_fail):
-    #     # status: 4 - expired (temp failure)
-    #     status = "4"
-    # elif to.strip().endswith(perm_fail):
-    #     # status: 5 - rejected (perm failure)
-    #     status = "5"
-    # else:
-    #     # status: 3 - delivered
-    #     status = "3"
+    if to.strip().endswith(temp_fail):
+        status = "4"
+    elif to.strip().endswith(perm_fail):
+        status = "5"
+    else:
+        status = "3"
 
-    # return json.dumps({"reference": "mmg_reference",
-    #                    "CID": str(notification_id),
-    #                    "MSISDN": to,
-    #                    "status": status,
-    #                    "deliverytime": "2016-04-05 16:01:07"})
+    return json.dumps({"reference": "mmg_reference",
+                       "CID": str(notification_id),
+                       "MSISDN": to,
+                       "status": status,
+                       "deliverytime": "2016-04-05 16:01:07"})
+
+
+def firetext_callback(notification_id, to):
+    """
+        status: 0 - delivered
+        status: 1 - perm failure
+    """
+    if to.strip().endswith(perm_fail):
+        status = "1"
+    elif to.strip().endswith(temp_fail):
+        status = "2"
+    else:
+        status = "0"
+    return {
+        'mobile': to,
+        'status': status,
+        'time': '2016-03-10 14:17:00',
+        'reference': notification_id
+    }
+
+
+@notify_celery.task(bind=True, name="create-fake-letter-response-file", max_retries=5, default_retry_delay=300)
+def create_fake_letter_response_file(self, reference):
+    now = datetime.utcnow()
+    dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
+
+    # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds
+    for i in sorted(range(30), key=lambda _: random.random()):
+        upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S'))
+        if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name):
+            break
+    else:
+        raise ValueError(
+            'cant create fake letter response file for {} - too many files for that time already exist on s3'.format(
+                reference
+            )
+        )
+
+    s3upload(
+        filedata=dvla_response_data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+        file_location=upload_file_name
+    )
+    current_app.logger.info("Fake DVLA response file {}, content [{}], uploaded to {}, created at {}".format(
+        upload_file_name, dvla_response_data, current_app.config['DVLA_RESPONSE_BUCKET_NAME'], now))
+
+    # on development we can't trigger SNS callbacks so we need to manually hit the DVLA callback endpoint
+    if current_app.config['NOTIFY_ENVIRONMENT'] == 'development':
+        make_request('letter', 'dvla', _fake_sns_s3_callback(upload_file_name), None)
+
+
+def _fake_sns_s3_callback(filename):
+    message_contents = '{"Records":[{"s3":{"object":{"key":"%s"}}}]}' % (filename)  # noqa
+    return json.dumps({
+        "Type": "Notification",
+        "MessageId": "some-message-id",
+        "Message": message_contents
+    })
 
 
 def ses_notification_callback(reference):
@@ -86,7 +167,7 @@ def ses_notification_callback(reference):
             'processingTimeMillis': 2003,
             'recipients': ['success@simulator.amazonses.com'],
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'a7-32.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'a7-32.smtp-out.eu-west-1.amazonses.com',
             'smtpResponse': '250 2.6.0 Message received',
             'timestamp': '2017-11-17T12:14:03.646Z'
         },
@@ -123,7 +204,7 @@ def ses_notification_callback(reference):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:01.643Z'
         },
@@ -133,14 +214,14 @@ def ses_notification_callback(reference):
     return {
         'Type': 'Notification',
         'MessageId': '8e83c020-1234-1234-1234-92a8ee9baa0a',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:03.710Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REACTED]',
         'MessageAttributes': {}
     }
 
@@ -166,7 +247,7 @@ def _ses_bounce_callback(reference, bounce_type):
             }],
             'feedbackId': '0102015fc9e676fb-12341234-1234-1234-1234-9301e86a4fa8-000000',
             'remoteMtaIp': '123.123.123.123',
-            'reportingMTA': 'dsn; a7-31.smtp-out.us-west-2.amazonses.com',
+            'reportingMTA': 'dsn; a7-31.smtp-out.eu-west-1.amazonses.com',
             'timestamp': '2017-11-17T12:14:05.131Z'
         },
         'mail': {
@@ -202,7 +283,7 @@ def _ses_bounce_callback(reference, bounce_type):
             'messageId': reference,
             'sendingAccountId': '12341234',
             'source': '"TEST" <TEST@notify.works>',
-            'sourceArn': 'arn:aws:ses:us-west-2:12341234:identity/notify.works',
+            'sourceArn': 'arn:aws:ses:eu-west-1:12341234:identity/notify.works',
             'sourceIp': '0.0.0.1',
             'timestamp': '2017-11-17T12:14:03.000Z'
         },
@@ -211,13 +292,13 @@ def _ses_bounce_callback(reference, bounce_type):
     return {
         'Type': 'Notification',
         'MessageId': '36e67c28-1234-1234-1234-2ea0172aa4a7',
-        'TopicArn': 'arn:aws:sns:us-west-2:12341234:ses_notifications',
+        'TopicArn': 'arn:aws:sns:eu-west-1:12341234:ses_notifications',
         'Subject': None,
         'Message': json.dumps(ses_message_body),
         'Timestamp': '2017-11-17T12:14:05.149Z',
         'SignatureVersion': '1',
         'Signature': '[REDACTED]',  # noqa
-        'SigningCertUrl': 'https://sns.us-west-2.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
-        'UnsubscribeUrl': 'https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
+        'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-[REDACTED]].pem',
+        'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=[REDACTED]]',
         'MessageAttributes': {}
     }

app/celery/scheduled_tasks.py

@@ -1,53 +1,57 @@
-from datetime import datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta
+)
 
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import (
-    NotifySupportTicket,
-)
-from sqlalchemy import between
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import and_
 from sqlalchemy.exc import SQLAlchemyError
 
 from app import notify_celery, zendesk_client
 from app.celery.tasks import (
-    get_recipient_csv_and_template_and_sender_id,
-    process_incomplete_jobs,
     process_job,
-    process_row,
-)
-from app.config import QueueNames
-from app.dao.invited_org_user_dao import (
-    delete_org_invitations_created_more_than_two_days_ago,
-)
-from app.dao.invited_user_dao import (
-    delete_invitations_created_more_than_two_days_ago,
+    get_recipient_csv_and_template_and_sender_id,
+    process_row
 )
+from app.celery.letters_pdf_tasks import create_letters_pdf
+from app.config import QueueNames, TaskNames
+from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
     find_jobs_with_missing_rows,
-    find_missing_row_for_job,
+    find_missing_row_for_job
+)
+from app.dao.jobs_dao import dao_update_job
+from app.dao.notifications_dao import (
+    dao_get_scheduled_notifications,
+    set_scheduled_notification_to_processed,
+    notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket,
+    is_delivery_slow_for_providers,
 )
-from app.dao.notifications_dao import notifications_not_yet_sent
 from app.dao.provider_details_dao import (
-    dao_adjust_provider_priority_back_to_resting_points,
-)
-from app.dao.services_dao import (
-    dao_find_services_sending_to_tv_numbers,
-    dao_find_services_with_high_failure_rates,
+    dao_reduce_sms_provider_priority,
+    dao_adjust_provider_priority_back_to_resting_points
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.dao.services_dao import dao_find_services_sending_to_tv_numbers, dao_find_services_with_high_failure_rates
 from app.models import (
-    EMAIL_TYPE,
-    JOB_STATUS_ERROR,
-    JOB_STATUS_IN_PROGRESS,
-    JOB_STATUS_PENDING,
-    SMS_TYPE,
     Job,
+    JOB_STATUS_IN_PROGRESS,
+    JOB_STATUS_ERROR,
+    SMS_TYPE,
+    EMAIL_TYPE,
 )
 from app.notifications.process_notifications import send_notification_to_queue
+from app.v2.errors import JobIncompleteError
 
 
 @notify_celery.task(name="run-scheduled-jobs")
+@statsd(namespace="tasks")
 def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
@@ -58,7 +62,23 @@ def run_scheduled_jobs():
         raise
 
 
+@notify_celery.task(name='send-scheduled-notifications')
+@statsd(namespace="tasks")
+def send_scheduled_notifications():
+    try:
+        scheduled_notifications = dao_get_scheduled_notifications()
+        for notification in scheduled_notifications:
+            send_notification_to_queue(notification, notification.service.research_mode)
+            set_scheduled_notification_to_processed(notification.id)
+        current_app.logger.info(
+            "Sent {} scheduled notifications to the provider queue".format(len(scheduled_notifications)))
+    except SQLAlchemyError:
+        current_app.logger.exception("Failed to send scheduled notifications")
+        raise
+
+
 @notify_celery.task(name="delete-verify-codes")
+@statsd(namespace="tasks")
 def delete_verify_codes():
     try:
         start = datetime.utcnow()
@@ -72,6 +92,7 @@ def delete_verify_codes():
 
 
 @notify_celery.task(name="delete-invitations")
+@statsd(namespace="tasks")
 def delete_invitations():
     try:
         start = datetime.utcnow()
@@ -85,43 +106,56 @@ def delete_invitations():
         raise
 
 
+@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
+def switch_current_sms_provider_on_slow_delivery():
+    """
+    Reduce provider's priority if at least 30% of notifications took more than four minutes to be delivered
+    in the last ten minutes. If both providers are slow, don't do anything. If we changed the providers in the
+    last ten minutes, then don't update them again either.
+    """
+    slow_delivery_notifications = is_delivery_slow_for_providers(
+        threshold=0.3,
+        created_at=datetime.utcnow() - timedelta(minutes=10),
+        delivery_time=timedelta(minutes=4),
+    )
+
+    # only adjust if some values are true and some are false - ie, don't adjust if all providers are fast or
+    # all providers are slow
+    if len(set(slow_delivery_notifications.values())) != 1:
+        for provider_name, is_slow in slow_delivery_notifications.items():
+            if is_slow:
+                current_app.logger.warning('Slow delivery notifications detected for provider {}'.format(provider_name))
+                dao_reduce_sms_provider_priority(provider_name, time_threshold=timedelta(minutes=10))
+
+
 @notify_celery.task(name='tend-providers-back-to-middle')
+@statsd(namespace='tasks')
 def tend_providers_back_to_middle():
     dao_adjust_provider_priority_back_to_resting_points()
 
 
 @notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
 def check_job_status():
     """
     every x minutes do this check
     select
     from jobs
     where job_status == 'in progress'
-    and processing started between 30 and 35 minutes ago
-    OR where the job_status == 'pending'
-    and the job scheduled_for timestamp is between 30 and 35 minutes ago.
+    and template_type in ('sms', 'email')
+    and scheduled_at or created_at is older that 30 minutes.
     if any results then
-        update the job_status to 'error'
+        raise error
         process the rows in the csv that are missing (in another task) just do the check here.
     """
     thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
     thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
 
-    incomplete_in_progress_jobs = Job.query.filter(
+    jobs_not_complete_after_30_minutes = Job.query.filter(
         Job.job_status == JOB_STATUS_IN_PROGRESS,
-        between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-    incomplete_pending_jobs = Job.query.filter(
-        Job.job_status == JOB_STATUS_PENDING,
-        Job.scheduled_for.isnot(None),
-        between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago)
-    )
-
-    jobs_not_complete_after_30_minutes = incomplete_in_progress_jobs.union(
-        incomplete_pending_jobs
-    ).order_by(
-        Job.processing_started, Job.scheduled_for
-    ).all()
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
+    ).order_by(Job.processing_started).all()
 
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
@@ -132,14 +166,16 @@ def check_job_status():
         job_ids.append(str(job.id))
 
     if job_ids:
-        current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
-        process_incomplete_jobs.apply_async(
-            [job_ids],
+        notify_celery.send_task(
+            name=TaskNames.PROCESS_INCOMPLETE_JOBS,
+            args=(job_ids,),
             queue=QueueNames.JOBS
         )
+        raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
 
 
 @notify_celery.task(name='replay-created-notifications')
+@statsd(namespace="tasks")
 def replay_created_notifications():
     # if the notification has not be send after 1 hour, then try to resend.
     resend_created_notifications_older_than = (60 * 60)
@@ -157,21 +193,80 @@ def replay_created_notifications():
         for n in notifications_to_resend:
             send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
 
+    # if the letter has not be send after an hour, then create a zendesk ticket
+    letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than)
+
+    if len(letters) > 0:
+        msg = "{} letters were created over an hour ago, " \
+              "but do not have an updated_at timestamp or billable units. " \
+              "\n Creating app.celery.letters_pdf_tasks.create_letters tasks to upload letter to S3 " \
+              "and update notifications for the following notification ids: " \
+              "\n {}".format(len(letters), [x.id for x in letters])
+
+        current_app.logger.info(msg)
+        for letter in letters:
+            create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)
+
+
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()
+
+    if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
+        msg = "{} precompiled letters have been pending-virus-check for over 90 minutes. " \
+              "Notifications: {}".format(len(letters), letter_ids)
+
+        current_app.logger.exception(msg)
+
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=msg,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+
+
+@notify_celery.task(name='check-templated-letter-state')
+@statsd(namespace="tasks")
+def check_templated_letter_state():
+    letters = dao_old_letters_with_created_status()
+
+    if len(letters) > 0:
+        letter_ids = [str(letter.id) for letter in letters]
+
+        msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
+              "Notifications: {}".format(len(letters), letter_ids)
+
+        current_app.logger.exception(msg)
+
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=msg,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+
 
 @notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
 def check_for_missing_rows_in_completed_jobs():
-    jobs = find_jobs_with_missing_rows()
-    for job in jobs:
-        recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
+    jobs_and_job_size = find_jobs_with_missing_rows()
+    for x in jobs_and_job_size:
+        job = x[1]
         missing_rows = find_missing_row_for_job(job.id, job.notification_count)
         for row_to_process in missing_rows:
-            row = recipient_csv[row_to_process.missing_row]
-            current_app.logger.info(
-                "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
-            process_row(row, template, job, job.service, sender_id=sender_id)
+            recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
+            for row in recipient_csv.get_rows():
+                if row.index == row_to_process.missing_row:
+                    current_app.logger.info(
+                        "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id))
+                    process_row(row, template, job, job.service, sender_id=sender_id)
 
 
 @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')
+@statsd(namespace="tasks")
 def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
     start_date = (datetime.utcnow() - timedelta(days=1))
     end_date = datetime.utcnow()
@@ -209,10 +304,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
         if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
             message += ("\nYou can find instructions for this ticket in our manual:\n"
                         "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers")  # noqa
-            ticket = NotifySupportTicket(
-                subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] High failure rates for sms spotted for services",
+            zendesk_client.create_ticket(
+                subject="[{}] High failure rates for sms spotted for services".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']
+                ),
                 message=message,
-                ticket_type=NotifySupportTicket.TYPE_INCIDENT,
-                technical_ticket=True
+                ticket_type=zendesk_client.TYPE_INCIDENT
             )
-            zendesk_client.send_ticket_to_zendesk(ticket)

app/celery/service_callback_tasks.py

@@ -1,14 +1,22 @@
 import json
 
 from flask import current_app
-from requests import HTTPError, RequestException, request
+from notifications_utils.statsd_decorators import statsd
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
 
-from app import encryption, notify_celery
+from app import (
+    notify_celery,
+    encryption
+)
 from app.config import QueueNames
-from app.utils import DATETIME_FORMAT
 
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
@@ -22,11 +30,8 @@ def send_delivery_status_to_service(
         "created_at": status_update['notification_created_at'],
         "completed_at": status_update['notification_updated_at'],
         "sent_at": status_update['notification_sent_at'],
-        "notification_type": status_update['notification_type'],
-        "template_id": status_update['template_id'],
-        "template_version": status_update['template_version']
+        "notification_type": status_update['notification_type']
     }
-
     _send_data_to_service_callback_api(
         self,
         data,
@@ -37,6 +42,7 @@ def send_delivery_status_to_service(
 
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_complaint_to_service(self, complaint_data):
     complaint = encryption.decrypt(complaint_data)
 
@@ -68,7 +74,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 'Content-Type': 'application/json',
                 'Authorization': 'Bearer {}'.format(token)
             },
-            timeout=5
+            timeout=60
         )
         current_app.logger.info('{} sending {} to {}, response {}'.format(
             function_name,
@@ -79,16 +85,16 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            "{} request failed for notification_id: {} and url: {}. exception: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
                 function_name,
                 notification_id,
                 service_callback_url,
                 e
             )
         )
-        if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
+        if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
-                self.retry(queue=QueueNames.CALLBACKS_RETRY)
+                self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.warning(
                     "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
@@ -97,24 +103,15 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                         notification_id
                     )
                 )
-        else:
-            current_app.logger.warning(
-                "{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
-                    function_name,
-                    notification_id,
-                    service_callback_url,
-                    e
-                )
-            )
 
 
 def create_delivery_status_callback_data(notification, service_callback_api):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "notification_id": str(notification.id),
         "notification_client_reference": notification.client_reference,
         "notification_to": notification.to,
         "notification_status": notification.status,
-        "notification_provider_response": notification.provider_response,  # TODO do we test for provider_response?
         "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
         "notification_updated_at":
             notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
@@ -122,13 +119,12 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_type": notification.notification_type,
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
-        "template_id": str(notification.template_id),
-        "template_version": notification.template_version,
     }
     return encryption.encrypt(data)
 
 
 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
     data = {
         "complaint_id": str(complaint.id),
         "notification_id": str(notification.id),

app/celery/tasks.py

@@ -1,43 +1,79 @@
 import json
 from datetime import datetime
+from collections import namedtuple, defaultdict
 
 from flask import current_app
-from notifications_utils.recipients import RecipientCSV
-from requests import HTTPError, RequestException, request
-from sqlalchemy.exc import IntegrityError, SQLAlchemyError
-
-from app import create_uuid, encryption, notify_celery
-from app.aws import s3
-from app.celery import provider_tasks
-from app.config import QueueNames
-from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
-from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
-from app.dao.notifications_dao import (
-    dao_get_last_notification_added_for_job_id,
-    get_notification_by_id,
+from notifications_utils.recipients import (
+    RecipientCSV
 )
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.template import (
+    SMSMessageTemplate,
+    WithSubjectTemplate,
+)
+from notifications_utils.timezones import convert_utc_to_bst
+from requests import (
+    HTTPError,
+    request,
+    RequestException
+)
+from sqlalchemy.exc import SQLAlchemyError
+
+from app import (
+    create_uuid,
+    create_random_identifier,
+    DATETIME_FORMAT,
+    encryption,
+    notify_celery,
+)
+from app.aws import s3
+from app.celery import provider_tasks, letters_pdf_tasks, research_mode_tasks
+from app.config import QueueNames
+from app.dao.daily_sorted_letter_dao import dao_create_or_update_daily_sorted_letter
+from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
+from app.dao.jobs_dao import (
+    dao_update_job,
+    dao_get_job_by_id,
+)
+from app.dao.notifications_dao import (
+    get_notification_by_id,
+    dao_update_notifications_by_reference,
+    dao_get_last_notification_added_for_job_id,
+    update_notification_status_by_reference,
+    dao_get_notification_history_by_reference,
+)
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
+from app.dao.returned_letters_dao import insert_or_update_returned_letters
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
+from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
+    DVLA_RESPONSE_STATUS_SENT,
     EMAIL_TYPE,
     JOB_STATUS_CANCELLED,
     JOB_STATUS_FINISHED,
     JOB_STATUS_IN_PROGRESS,
     JOB_STATUS_PENDING,
     KEY_TYPE_NORMAL,
+    LETTER_TYPE,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
     SMS_TYPE,
+    DailySortedLetter,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
-from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT
-from app.v2.errors import TooManyRequestsError
 
 
 @notify_celery.task(name="process-job")
+@statsd(namespace="tasks")
 def process_job(job_id, sender_id=None):
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
@@ -48,10 +84,6 @@ def process_job(job_id, sender_id=None):
 
     service = job.service
 
-    job.job_status = JOB_STATUS_IN_PROGRESS
-    job.processing_started = start
-    dao_update_job(job)
-
     if not service.active:
         job.job_status = JOB_STATUS_CANCELLED
         dao_update_job(job)
@@ -62,6 +94,10 @@ def process_job(job_id, sender_id=None):
     if __sending_limits_for_job_exceeded(service, job, job_id):
         return
 
+    job.job_status = JOB_STATUS_IN_PROGRESS
+    job.processing_started = start
+    dao_update_job(job)
+
     recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
 
     current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
@@ -91,10 +127,13 @@ def job_complete(job, resumed=False, start=None):
 
 def get_recipient_csv_and_template_and_sender_id(job):
     db_template = dao_get_template_by_id(job.template_id, job.template_version)
-    template = db_template._as_utils_template()
 
+    TemplateClass = get_template_class(db_template.template_type)
+    template = TemplateClass(db_template.__dict__)
     contents, meta_data = s3.get_job_and_metadata_from_s3(service_id=str(job.service_id), job_id=str(job.id))
-    recipient_csv = RecipientCSV(contents, template=template)
+    recipient_csv = RecipientCSV(file_data=contents,
+                                 template_type=template.template_type,
+                                 placeholders=template.placeholders)
 
     return recipient_csv, template, meta_data.get("sender_id")
 
@@ -112,7 +151,8 @@ def process_row(row, template, job, service, sender_id=None):
 
     send_fns = {
         SMS_TYPE: save_sms,
-        EMAIL_TYPE: save_email
+        EMAIL_TYPE: save_email,
+        LETTER_TYPE: save_letter
     }
 
     send_fn = send_fns[template_type]
@@ -135,13 +175,9 @@ def process_row(row, template, job, service, sender_id=None):
 
 
 def __sending_limits_for_job_exceeded(service, job, job_id):
-    try:
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-        if total_sent + job.notification_count > service.message_limit:
-            raise TooManyRequestsError(service.message_limit)
-        else:
-            return False
-    except TooManyRequestsError:
+    total_sent = fetch_todays_total_message_count(service.id)
+
+    if total_sent + job.notification_count > service.message_limit:
         job.job_status = 'sending limits exceeded'
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
@@ -150,26 +186,24 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
                 job_id, job.notification_count, service.message_limit)
         )
         return True
+    return False
 
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_sms(self,
              service_id,
              notification_id,
              encrypted_notification,
              sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
 
     if sender_id:
         reply_to_text = dao_get_service_sms_senders_by_id(service_id, sender_id).sms_sender
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.debug(
@@ -211,6 +245,7 @@ def save_sms(self,
 
 
 @notify_celery.task(bind=True, name="save-email", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def save_email(self,
                service_id,
                notification_id,
@@ -218,17 +253,13 @@ def save_email(self,
                sender_id=None):
     notification = encryption.decrypt(encrypted_notification)
 
-    service = SerialisedService.from_id(service_id)
-    template = SerialisedTemplate.from_id_and_service_id(
-        notification['template'],
-        service_id=service.id,
-        version=notification['template_version'],
-    )
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
 
     if sender_id:
         reply_to_text = dao_get_reply_to_by_id(service_id, sender_id).email_address
     else:
-        reply_to_text = template.reply_to_text
+        reply_to_text = template.get_reply_to_text()
 
     if not service_allowed_to_send_to(notification['to'], service, KEY_TYPE_NORMAL):
         current_app.logger.info("Email {} failed as restricted service".format(notification_id))
@@ -261,59 +292,98 @@ def save_email(self,
         handle_exception(self, notification, notification_id, e)
 
 
-@notify_celery.task(bind=True, name="save-api-email", max_retries=5, default_retry_delay=300)
-def save_api_email(self, encrypted_notification):
-
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-@notify_celery.task(bind=True, name="save-api-sms", max_retries=5, default_retry_delay=300)
-def save_api_sms(self, encrypted_notification):
-    save_api_email_or_sms(self, encrypted_notification)
-
-
-def save_api_email_or_sms(self, encrypted_notification):
+@notify_celery.task(bind=True, name="save-letter", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
+def save_letter(
+        self,
+        service_id,
+        notification_id,
+        encrypted_notification,
+):
     notification = encryption.decrypt(encrypted_notification)
-    service = SerialisedService.from_id(notification['service_id'])
-    q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
-    provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
-        else provider_tasks.deliver_sms
+
+    # we store the recipient as just the first item of the person's address
+    recipient = notification['personalisation']['addressline1']
+
+    service = dao_fetch_service_by_id(service_id)
+    template = dao_get_template_by_id(notification['template'], version=notification['template_version'])
+
     try:
+        # if we don't want to actually send the letter, then start it off in SENDING so we don't pick it up
+        status = NOTIFICATION_CREATED if not service.research_mode else NOTIFICATION_SENDING
 
-        persist_notification(
-            notification_id=notification["id"],
-            template_id=notification['template_id'],
+        saved_notification = persist_notification(
+            template_id=notification['template'],
             template_version=notification['template_version'],
-            recipient=notification['to'],
+            template_postage=template.postage,
+            recipient=recipient,
             service=service,
-            personalisation=notification.get('personalisation'),
-            notification_type=notification['notification_type'],
-            client_reference=notification['client_reference'],
-            api_key_id=notification.get('api_key_id'),
+            personalisation=notification['personalisation'],
+            notification_type=LETTER_TYPE,
+            api_key_id=None,
             key_type=KEY_TYPE_NORMAL,
-            created_at=notification['created_at'],
-            reply_to_text=notification['reply_to_text'],
-            status=notification['status'],
-            document_download_count=notification['document_download_count']
+            created_at=datetime.utcnow(),
+            job_id=notification['job'],
+            job_row_number=notification['row_number'],
+            notification_id=notification_id,
+            reference=create_random_identifier(),
+            reply_to_text=template.get_reply_to_text(),
+            status=status
         )
 
-        q = q if not service.research_mode else QueueNames.RESEARCH_MODE
-        provider_task.apply_async(
-            [notification['id']],
-            queue=q
-        )
-        current_app.logger.debug(
-            f"{notification['notification_type']} {notification['id']} has been persisted and sent to delivery queue."
-        )
-    except IntegrityError:
-        current_app.logger.info(f"{notification['notification_type']} {notification['id']} already exists.")
+        if not service.research_mode:
+            letters_pdf_tasks.create_letters_pdf.apply_async(
+                [str(saved_notification.id)],
+                queue=QueueNames.CREATE_LETTERS_PDF
+            )
+        elif current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']:
+            research_mode_tasks.create_fake_letter_response_file.apply_async(
+                (saved_notification.reference,),
+                queue=QueueNames.RESEARCH_MODE
+            )
+        else:
+            update_notification_status_by_reference(saved_notification.reference, 'delivered')
 
-    except SQLAlchemyError:
+        current_app.logger.debug("Letter {} created at {}".format(saved_notification.id, saved_notification.created_at))
+    except SQLAlchemyError as e:
+        handle_exception(self, notification, notification_id, e)
 
-        try:
-            self.retry(queue=QueueNames.RETRY)
-        except self.MaxRetriesExceededError:
-            current_app.logger.error(f"Max retry failed Failed to persist notification {notification['id']}")
+
+@notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
+@statsd(namespace="tasks")
+def update_letter_notifications_to_sent_to_dvla(self, notification_references):
+    # This task will be called by the FTP app to update notifications as sent to DVLA
+    provider = get_provider_details_by_notification_type(LETTER_TYPE)[0]
+
+    updated_count, _ = dao_update_notifications_by_reference(
+        notification_references,
+        {
+            'status': NOTIFICATION_SENDING,
+            'sent_by': provider.identifier,
+            'sent_at': datetime.utcnow(),
+            'updated_at': datetime.utcnow()
+        }
+    )
+
+    current_app.logger.info("Updated {} letter notifications to sending".format(updated_count))
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-to-error')
+@statsd(namespace="tasks")
+def update_letter_notifications_to_error(self, notification_references):
+    # This task will be called by the FTP app to update notifications as sent to DVLA
+
+    updated_count, _ = dao_update_notifications_by_reference(
+        notification_references,
+        {
+            'status': NOTIFICATION_TECHNICAL_FAILURE,
+            'updated_at': datetime.utcnow()
+        }
+    )
+    message = "Updated {} letter notifications to technical-failure with references {}".format(
+        updated_count, notification_references
+    )
+    raise NotificationTechnicalFailureException(message)
 
 
 def handle_exception(task, notification, notification_id, exc):
@@ -327,7 +397,6 @@ def handle_exception(task, notification, notification_id, exc):
         # Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
         # SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
         # send to the retry queue.
-        # This probably (hopefully) is not an issue with Redis as the celery backing store
         current_app.logger.exception('Retry' + retry_msg)
         try:
             task.retry(queue=QueueNames.RETRY, exc=exc)
@@ -335,7 +404,121 @@ def handle_exception(task, notification, notification_id, exc):
             current_app.logger.error('Max retry failed' + retry_msg)
 
 
+def get_template_class(template_type):
+    if template_type == SMS_TYPE:
+        return SMSMessageTemplate
+    elif template_type in (EMAIL_TYPE, LETTER_TYPE):
+        # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
+        # use the same base template
+        return WithSubjectTemplate
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
+def update_letter_notifications_statuses(self, filename):
+    notification_updates = parse_dvla_file(filename)
+
+    temporary_failures = []
+
+    for update in notification_updates:
+        check_billable_units(update)
+        update_letter_notification(filename, temporary_failures, update)
+    if temporary_failures:
+        # This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
+        message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \
+            .format(filename=filename, failures=temporary_failures)
+        raise DVLAException(message)
+
+
+@notify_celery.task(bind=True, name="record-daily-sorted-counts")
+@statsd(namespace="tasks")
+def record_daily_sorted_counts(self, filename):
+    sorted_letter_counts = defaultdict(int)
+    notification_updates = parse_dvla_file(filename)
+    for update in notification_updates:
+        sorted_letter_counts[update.cost_threshold.lower()] += 1
+
+    unknown_status = sorted_letter_counts.keys() - {'unsorted', 'sorted'}
+    if unknown_status:
+        message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
+            filename, unknown_status.__repr__()
+        )
+        raise DVLAException(message)
+
+    billing_date = get_billing_date_in_bst_from_filename(filename)
+    persist_daily_sorted_letter_counts(day=billing_date,
+                                       file_name=filename,
+                                       sorted_letter_counts=sorted_letter_counts)
+
+
+def parse_dvla_file(filename):
+    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
+    response_file_content = s3.get_s3_file(bucket_location, filename)
+
+    try:
+        return process_updates_from_file(response_file_content)
+    except TypeError:
+        raise DVLAException('DVLA response file: {} has an invalid format'.format(filename))
+
+
+def get_billing_date_in_bst_from_filename(filename):
+    # exclude seconds from the date since we don't need it. We got a date ending in 60 second - which is not valid.
+    datetime_string = filename.split('-')[1][:-2]
+    datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M')
+    return convert_utc_to_bst(datetime_obj).date()
+
+
+def persist_daily_sorted_letter_counts(day, file_name, sorted_letter_counts):
+    daily_letter_count = DailySortedLetter(
+        billing_day=day,
+        file_name=file_name,
+        unsorted_count=sorted_letter_counts['unsorted'],
+        sorted_count=sorted_letter_counts['sorted']
+    )
+    dao_create_or_update_daily_sorted_letter(daily_letter_count)
+
+
+def process_updates_from_file(response_file):
+    NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
+    notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
+    return notification_updates
+
+
+def update_letter_notification(filename, temporary_failures, update):
+    if update.status == DVLA_RESPONSE_STATUS_SENT:
+        status = NOTIFICATION_DELIVERED
+    else:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+        temporary_failures.append(update.reference)
+
+    updated_count, _ = dao_update_notifications_by_reference(
+        references=[update.reference],
+        update_dict={"status": status,
+                     "updated_at": datetime.utcnow()
+                     }
+    )
+
+    if not updated_count:
+        msg = "Update letter notification file {filename} failed: notification either not found " \
+              "or already updated from delivered. Status {status} for notification reference {reference}".format(
+                  filename=filename, status=status, reference=update.reference)
+        current_app.logger.info(msg)
+
+
+def check_billable_units(notification_update):
+    notification = dao_get_notification_history_by_reference(notification_update.reference)
+
+    if int(notification_update.page_count) != notification.billable_units:
+        msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(
+            notification.id, notification.billable_units, notification_update.page_count)
+        try:
+            raise DVLAException(msg)
+        except DVLAException:
+            current_app.logger.exception(msg)
+
+
 @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
 def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
     inbound_api = get_service_inbound_api_for_service(service_id=service_id)
     if not inbound_api:
@@ -364,32 +547,36 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
             },
             timeout=60
         )
-        current_app.logger.debug(
-            f"send_inbound_sms_to_service sending {inbound_sms_id} to {inbound_api.url}, " +
-            f"response {response.status_code}"
-        )
+        current_app.logger.debug('send_inbound_sms_to_service sending {} to {}, response {}'.format(
+            inbound_sms_id,
+            inbound_api.url,
+            response.status_code
+        ))
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            f"send_inbound_sms_to_service failed for service_id: {service_id} for inbound_sms_id: {inbound_sms_id} " +
-            f"and url: {inbound_api.url}. exception: {e}"
+            "send_inbound_sms_to_service failed for service_id: {} for inbound_sms_id: {} and url: {}. exc: {}".format(
+                service_id,
+                inbound_sms_id,
+                inbound_api.url,
+                e
+            )
         )
         if not isinstance(e, HTTPError) or e.response.status_code >= 500:
             try:
                 self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.error(
-                    "Retry: send_inbound_sms_to_service has retried the max number of" +
-                    f"times for service: {service_id} and inbound_sms {inbound_sms_id}"
+                    """Retry: send_inbound_sms_to_service has retried the max number of
+                     times for service: {} and  inbound_sms {}""".format(
+                        service_id,
+                        inbound_sms_id
+                    )
                 )
-        else:
-            current_app.logger.warning(
-                f"send_inbound_sms_to_service is not being retried for service_id: {service_id} for " +
-                f"inbound_sms id: {inbound_sms_id} and url: {inbound_api.url}. exception: {e}"
-            )
 
 
 @notify_celery.task(name='process-incomplete-jobs')
+@statsd(namespace="tasks")
 def process_incomplete_jobs(job_ids):
     jobs = [dao_get_job_by_id(job_id) for job_id in job_ids]
 
@@ -423,3 +610,20 @@ def process_incomplete_job(job_id):
             process_row(row, template, job, job.service, sender_id=sender_id)
 
     job_complete(job, resumed=True)
+
+
+@notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
+def process_returned_letters_list(notification_references):
+    updated, updated_history = dao_update_notifications_by_reference(
+        notification_references,
+        {"status": NOTIFICATION_RETURNED_LETTER}
+    )
+
+    insert_or_update_returned_letters(notification_references)
+
+    current_app.logger.info(
+        "Updated {} letter notifications ({} history notifications, from {} references) to returned-letter".format(
+            updated, updated_history, len(notification_references)
+        )
+    )

app/clients/__init__.py

@@ -17,7 +17,7 @@ STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'
 
 
-class NotificationProviderClients(object):
+class Clients(object):
     sms_clients = {}
     email_clients = {}
 
@@ -35,7 +35,7 @@ class NotificationProviderClients(object):
         return self.email_clients.get(name)
 
     def get_client_by_name_and_type(self, name, notification_type):
-        assert notification_type in ['email', 'sms']  # nosec B101
+        assert notification_type in ['email', 'sms']
 
         if notification_type == 'email':
             return self.get_email_client(name)

app/clients/document_download.py

@@ -1,4 +1,7 @@
+import base64
+
 import requests
+
 from flask import current_app
 
 
@@ -23,16 +26,15 @@ class DocumentDownloadClient:
     def get_upload_url(self, service_id):
         return "{}/services/{}/documents".format(self.api_host, service_id)
 
-    def upload_document(self, service_id, file_contents, is_csv=None):
+    def upload_document(self, service_id, file_contents):
         try:
             response = requests.post(
                 self.get_upload_url(service_id),
                 headers={
                     'Authorization': "Bearer {}".format(self.auth_token),
                 },
-                json={
-                    'document': file_contents,
-                    'is_csv': is_csv or False,
+                files={
+                    'document': base64.b64decode(file_contents)
                 }
             )
 
@@ -41,9 +43,7 @@ class DocumentDownloadClient:
             # if doc dl responds with a non-400, (eg 403) it's referring to credentials that the API and Doc DL use.
             # we don't want to tell users about that, so anything that isn't a 400 (virus scan failed or file type
             # unrecognised) should be raised as a 500 internal server error here.
-            if e.response is None:
-                raise Exception(f'Unhandled document download error: {repr(e)}')
-            elif e.response.status_code == 400:
+            if e.response.status_code == 400:
                 error = DocumentDownloadError.from_exception(e)
                 current_app.logger.info(
                     'Document download request failed with error: {}'.format(error.message)

app/clients/email/__init__.py

@@ -1,4 +1,4 @@
-from app.clients import Client, ClientException
+from app.clients import ClientException, Client
 
 
 class EmailClientException(ClientException):
@@ -8,18 +8,6 @@ class EmailClientException(ClientException):
     pass
 
 
-class EmailClientNonRetryableException(ClientException):
-    '''
-    Represents an error returned from the email client API with a 4xx response code
-    that should not be retried and should instead be marked as technical failure.
-    An example of this would be an email address that makes it through our
-    validation rules but is rejected by SES. There is no point in retrying this type as
-    it will always fail however many calls to SES. Whereas a throttling error would not
-    use this exception as it may succeed if we retry
-    '''
-    pass
-
-
 class EmailClient(Client):
     '''
     Base Email client for sending emails.
@@ -28,6 +16,5 @@ class EmailClient(Client):
     def send_email(self, *args, **kwargs):
         raise NotImplementedError('TODO Need to implement.')
 
-    @property
-    def name(self):
+    def get_name(self):
         raise NotImplementedError('TODO Need to implement.')

app/clients/email/aws_ses.py

@@ -1,16 +1,11 @@
-from time import monotonic
-
+import boto3
 import botocore
-from boto3 import client
 from flask import current_app
+from time import monotonic
+from notifications_utils.recipients import InvalidEmailError
 
 from app.clients import STATISTICS_DELIVERED, STATISTICS_FAILURE
-from app.clients.email import (
-    EmailClient,
-    EmailClientException,
-    EmailClientNonRetryableException,
-)
-from app.cloudfoundry_config import cloud_config
+from app.clients.email import (EmailClientException, EmailClient)
 
 ses_response_map = {
     'Permanent': {
@@ -48,28 +43,19 @@ class AwsSesClientException(EmailClientException):
     pass
 
 
-class AwsSesClientThrottlingSendRateException(AwsSesClientException):
-    pass
-
-
 class AwsSesClient(EmailClient):
     '''
     Amazon SES email client.
     '''
 
-    def init_app(self, statsd_client, *args, **kwargs):
-        self._client = client(
-            'ses',
-            region_name=cloud_config.ses_region,
-            aws_access_key_id=cloud_config.ses_access_key,
-            aws_secret_access_key=cloud_config.ses_secret_key
-        )
+    def init_app(self, region, statsd_client, *args, **kwargs):
+        self._client = boto3.client('ses', region_name=region)
         super(AwsSesClient, self).__init__(*args, **kwargs)
+        self.name = 'ses'
         self.statsd_client = statsd_client
 
-    @property
-    def name(self):
-        return 'ses'
+    def get_name(self):
+        return self.name
 
     def send_email(self,
                    source,
@@ -114,12 +100,10 @@ class AwsSesClient(EmailClient):
 
             # http://docs.aws.amazon.com/ses/latest/DeveloperGuide/api-error-codes.html
             if e.response['Error']['Code'] == 'InvalidParameterValue':
-                raise EmailClientNonRetryableException(e.response['Error']['Message'])
-            elif (
-                e.response['Error']['Code'] == 'Throttling'
-                and e.response['Error']['Message'] == 'Maximum sending rate exceeded.'
-            ):
-                raise AwsSesClientThrottlingSendRateException(str(e))
+                raise InvalidEmailError('email: "{}" message: "{}"'.format(
+                    to_addresses[0],
+                    e.response['Error']['Message']
+                ))
             else:
                 self.statsd_client.incr("clients.ses.error")
                 raise AwsSesClientException(str(e))

app/clients/email/aws_ses_stub.py

@@ -1,49 +0,0 @@
-import json
-from time import monotonic
-
-from flask import current_app
-from requests import request
-
-from app.clients.email import EmailClient, EmailClientException
-
-
-class AwsSesStubClientException(EmailClientException):
-    pass
-
-
-class AwsSesStubClient(EmailClient):
-    def init_app(self, statsd_client, stub_url):
-        self.statsd_client = statsd_client
-        self.url = stub_url
-
-    @property
-    def name(self):
-        return 'ses'
-
-    def send_email(self,
-                   source,
-                   to_addresses,
-                   subject,
-                   body,
-                   html_body='',
-                   reply_to_address=None):
-        try:
-            start_time = monotonic()
-            response = request(
-                "POST",
-                self.url,
-                data={"id": "dummy-data"},
-                timeout=60
-            )
-            response.raise_for_status()
-            response_json = json.loads(response.text)
-
-        except Exception as e:
-            self.statsd_client.incr("clients.ses_stub.error")
-            raise AwsSesStubClientException(str(e))
-        else:
-            elapsed_time = monotonic() - start_time
-            current_app.logger.info("AWS SES stub request finished in {}".format(elapsed_time))
-            self.statsd_client.timing("clients.ses_stub.request-time", elapsed_time)
-            self.statsd_client.incr("clients.ses_stub.success")
-            return response_json['MessageId']

app/clients/performance_platform/performance_platform_client.py

@@ -1,9 +1,10 @@
 import base64
 import json
 
-import requests
 from flask import current_app
-from notifications_utils.timezones import convert_utc_to_local_timezone
+import requests
+
+from notifications_utils.timezones import convert_utc_to_bst
 
 
 class PerformancePlatformClient:
@@ -55,7 +56,7 @@ class PerformancePlatformClient:
         :param period - the period that this data covers - "day", "week", "month", "quarter".
         """
         payload = {
-            '_timestamp': convert_utc_to_local_timezone(start_time).isoformat(),
+            '_timestamp': convert_utc_to_bst(start_time).isoformat(),
             'service': 'govuk-notify',
             'dataType': dataset,
             'period': period,

app/clients/sms/__init__.py

@@ -1,10 +1,10 @@
-from app.clients import Client, ClientException
+from app.clients import (Client, ClientException)
 
 
 class SmsClientResponseException(ClientException):
-    """
+    '''
     Base Exception for SmsClientsResponses
-    """
+    '''
 
     def __init__(self, message):
         self.message = message
@@ -14,15 +14,12 @@ class SmsClientResponseException(ClientException):
 
 
 class SmsClient(Client):
-    """
+    '''
     Base Sms client for sending smss.
-    """
-
-    def init_app(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+    '''
 
     def send_sms(self, *args, **kwargs):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')
 
     def get_name(self):
-        raise NotImplementedError("TODO Need to implement.")
+        raise NotImplementedError('TODO Need to implement.')

app/clients/sms/aws_sns.py

@@ -1,85 +0,0 @@
-import re
-from time import monotonic
-
-import botocore
-import phonenumbers
-from boto3 import client
-
-from app.clients.sms import SmsClient
-from app.cloudfoundry_config import cloud_config
-
-
-class AwsSnsClient(SmsClient):
-    """
-    AwsSns sms client
-    """
-
-    def init_app(self, current_app, statsd_client, *args, **kwargs):
-        self._client = client(
-            "sns",
-            region_name=cloud_config.sns_region,
-            aws_access_key_id=cloud_config.sns_access_key,
-            aws_secret_access_key=cloud_config.sns_secret_key
-        )
-        super(SmsClient, self).__init__(*args, **kwargs)
-        self.current_app = current_app
-        self.statsd_client = statsd_client
-        self._valid_sender_regex = re.compile(r"^\+?\d{5,14}$")
-
-    @property
-    def name(self):
-        return 'sns'
-
-    def get_name(self):
-        return self.name
-
-    def _valid_sender_number(self, sender):
-        return sender and re.match(self._valid_sender_regex, sender)
-
-    def send_sms(self, to, content, reference, sender=None, international=False):
-        matched = False
-
-        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
-            matched = True
-            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
-
-            # See documentation
-            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
-            attributes = {
-                "AWS.SNS.SMS.SMSType": {
-                    "DataType": "String",
-                    "StringValue": "Transactional",
-                }
-            }
-
-            if self._valid_sender_number(sender):
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": sender,
-                }
-            else:
-                attributes["AWS.MM.SMS.OriginationNumber"] = {
-                    "DataType": "String",
-                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
-                }
-
-            try:
-                start_time = monotonic()
-                response = self._client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
-            except botocore.exceptions.ClientError as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            except Exception as e:
-                self.statsd_client.incr("clients.sns.error")
-                raise str(e)
-            finally:
-                elapsed_time = monotonic() - start_time
-                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
-                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
-                self.statsd_client.incr("clients.sns.success")
-            return response["MessageId"]
-
-        if not matched:
-            self.statsd_client.incr("clients.sns.error")
-            self.current_app.logger.error("No valid numbers found in {}".format(to))
-            raise ValueError("No valid numbers found for SMS delivery")

app/clients/sms/firetext.py

@@ -0,0 +1,107 @@
+import json
+import logging
+
+from time import monotonic
+from requests import request, RequestException
+
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+logger = logging.getLogger(__name__)
+
+# Firetext will send a delivery receipt with three different status codes.
+# The `firetext_response` maps these codes to the notification statistics status and notification status.
+# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
+# the notification status to temporary-failure rather than permanent failure.
+#  See the code in the notification_dao.update_notifications_status_by_id
+firetext_responses = {
+    '0': 'delivered',
+    '1': 'permanent-failure',
+    '2': 'pending'
+}
+
+
+def get_firetext_responses(status):
+    return firetext_responses[status]
+
+
+class FiretextClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class FiretextClient(SmsClient):
+    '''
+    FireText sms client.
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('FIRETEXT_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'firetext'
+        self.url = current_app.config.get('FIRETEXT_URL')
+        self.statsd_client = statsd_client
+
+    def get_name(self):
+        return self.name
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.firetext.success")
+        else:
+            self.statsd_client.incr("clients.firetext.error")
+            self.current_app.logger.error(log_message)
+
+    def send_sms(self, to, content, reference, sender=None):
+
+        data = {
+            "apiKey": self.api_key,
+            "from": self.from_number if sender is None else sender,
+            "to": to.replace('+', ''),
+            "message": content,
+            "reference": reference
+        }
+
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.url,
+                data=data,
+                timeout=60
+            )
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+                if response.json()['code'] != 0:
+                    raise ValueError()
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise FiretextClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise FiretextClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.current_app.logger.info("Firetext request for {} finished in {}".format(reference, elapsed_time))
+            self.statsd_client.timing("clients.firetext.request-time", elapsed_time)
+        return response

app/clients/sms/mmg.py

@@ -0,0 +1,102 @@
+import json
+from time import monotonic
+from requests import (request, RequestException)
+from app.clients.sms import (SmsClient, SmsClientResponseException)
+
+mmg_response_map = {
+    '2': 'permanent-failure',
+    '3': 'delivered',
+    '4': 'temporary-failure',
+    '5': 'permanent-failure'
+}
+
+
+def get_mmg_responses(status):
+    return mmg_response_map[status]
+
+
+class MMGClientResponseException(SmsClientResponseException):
+    def __init__(self, response, exception):
+        status_code = response.status_code if response is not None else 504
+        text = response.text if response is not None else "Gateway Time-out"
+
+        self.status_code = status_code
+        self.text = text
+        self.exception = exception
+
+    def __str__(self):
+        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
+
+
+class MMGClient(SmsClient):
+    '''
+    MMG sms client
+    '''
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.api_key = current_app.config.get('MMG_API_KEY')
+        self.from_number = current_app.config.get('FROM_NUMBER')
+        self.name = 'mmg'
+        self.statsd_client = statsd_client
+        self.mmg_url = current_app.config.get('MMG_URL')
+
+    def record_outcome(self, success, response):
+        status_code = response.status_code if response else 503
+        log_message = "API {} request {} on {} response status_code {}".format(
+            "POST",
+            "succeeded" if success else "failed",
+            self.mmg_url,
+            status_code
+        )
+
+        if success:
+            self.current_app.logger.info(log_message)
+            self.statsd_client.incr("clients.mmg.success")
+        else:
+            self.statsd_client.incr("clients.mmg.error")
+            self.current_app.logger.error(log_message)
+
+    def get_name(self):
+        return self.name
+
+    def send_sms(self, to, content, reference, multi=True, sender=None):
+        data = {
+            "reqType": "BULK",
+            "MSISDN": to,
+            "msg": content,
+            "sender": self.from_number if sender is None else sender,
+            "cid": reference,
+            "multi": multi
+        }
+
+        start_time = monotonic()
+        try:
+            response = request(
+                "POST",
+                self.mmg_url,
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Basic {}'.format(self.api_key)
+                },
+                timeout=60
+            )
+
+            response.raise_for_status()
+            try:
+                json.loads(response.text)
+            except (ValueError, AttributeError) as e:
+                self.record_outcome(False, response)
+                raise MMGClientResponseException(response=response, exception=e)
+            self.record_outcome(True, response)
+        except RequestException as e:
+            self.record_outcome(False, e.response)
+            raise MMGClientResponseException(response=e.response, exception=e)
+        finally:
+            elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("clients.mmg.request-time", elapsed_time)
+            self.current_app.logger.info("MMG request for {} finished in {}".format(reference, elapsed_time))
+
+        return response

app/cloudfoundry_config.py

@@ -1,102 +1,21 @@
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
+import os
 import json
-from os import getenv
 
 
-class CloudfoundryConfig:
-    def __init__(self):
-        self.parsed_services = json.loads(getenv('VCAP_SERVICES') or '{}')
-        buckets = self.parsed_services.get('s3') or []
-        self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
-        self._empty_bucket_credentials = {
-            'bucket': '',
-            'access_key_id': '',
-            'secret_access_key': '',
-            'region': ''
-        }
-
-    @property
-    def database_url(self):
-        return getenv('DATABASE_URL', '').replace('postgres://', 'postgresql://')
-
-    @property
-    def redis_url(self):
-        try:
-            return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
-                'redis://',
-                'rediss://'
-            )
-        except KeyError:
-            return getenv('REDIS_URL')
-
-    def s3_credentials(self, service_name):
-        return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
-
-    @property
-    def ses_email_domain(self):
-        try:
-            domain_arn = self._ses_credentials('domain_arn')
-        except KeyError:
-            domain_arn = getenv('SES_DOMAIN_ARN', 'dev.notify.gov')
-        return domain_arn.split('/')[-1]
-
-    @property
-    def ses_region(self):
-        try:
-            return self._ses_credentials('region')
-        except KeyError:
-            return getenv('SES_AWS_REGION', 'us-west-1')
-
-    @property
-    def ses_access_key(self):
-        try:
-            return self._ses_credentials('smtp_user')
-        except KeyError:
-            return getenv('SES_AWS_ACCESS_KEY_ID')
-
-    @property
-    def ses_secret_key(self):
-        try:
-            return self._ses_credentials('secret_access_key')
-        except KeyError:
-            return getenv('SES_AWS_SECRET_ACCESS_KEY')
-
-    @property
-    def sns_access_key(self):
-        try:
-            return self._sns_credentials('aws_access_key_id')
-        except KeyError:
-            return getenv('SNS_AWS_ACCESS_KEY_ID')
-
-    @property
-    def sns_secret_key(self):
-        try:
-            return self._sns_credentials('aws_secret_access_key')
-        except KeyError:
-            return getenv('SNS_AWS_SECRET_ACCESS_KEY')
-
-    @property
-    def sns_region(self):
-        try:
-            return self._sns_credentials('region')
-        except KeyError:
-            return getenv('SNS_AWS_REGION', 'us-west-1')
-
-    @property
-    def sns_topic_arns(self):
-        try:
-            return [
-                self._ses_credentials('bounce_topic_arn'),
-                self._ses_credentials('complaint_topic_arn'),
-                self._ses_credentials('delivery_topic_arn')
-            ]
-        except KeyError:
-            return []
-
-    def _ses_credentials(self, key):
-        return self.parsed_services['datagov-smtp'][0]['credentials'][key]
-
-    def _sns_credentials(self, key):
-        return self.parsed_services['ttsnotify-sms'][0]['credentials'][key]
+def extract_cloudfoundry_config():
+    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+    set_config_env_vars(vcap_services)
 
 
-cloud_config = CloudfoundryConfig()
+def set_config_env_vars(vcap_services):
+    # Postgres config
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri']
+
+    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'

app/commands.py

@@ -1,29 +1,29 @@
 import csv
 import functools
-import itertools
 import uuid
 from datetime import datetime, timedelta
-from os import getenv
+from decimal import Decimal
 
 import click
 import flask
+import itertools
 from click_datetime import Datetime as click_dt
 from flask import current_app, json
-from notifications_python_client.authentication import create_jwt_token
 from notifications_utils.recipients import RecipientCSV
-from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
-from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from notifications_utils.statsd_decorators import statsd
 
-from app import db
+from app import db, DATETIME_FORMAT, encryption
 from app.aws import s3
-from app.celery.tasks import process_row
-from app.dao.annual_billing_dao import (
-    dao_create_or_update_annual_billing_for_year,
-    set_default_free_allowance_for_service,
-)
+from app.celery.tasks import record_daily_sorted_counts, get_template_class, process_row
+from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
+from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.celery.letters_pdf_tasks import create_letters_pdf
+from app.celery.reporting_tasks import create_nightly_notification_status_for_day
+from app.config import QueueNames
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
 from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
@@ -31,39 +31,35 @@ from app.dao.fact_billing_dao import (
     update_fact_billing,
 )
 from app.dao.jobs_dao import dao_get_job_by_id
-from app.dao.organisation_dao import (
-    dao_add_service_to_organisation,
-    dao_get_organisation_by_email_address,
-    dao_get_organisation_by_id,
-)
+from app.dao.organisation_dao import dao_get_organisation_by_email_address, dao_add_service_to_organisation
+
+from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
+from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.services_dao import (
-    dao_fetch_all_services_by_user,
-    dao_fetch_all_services_created_by_user,
-    dao_fetch_service_by_id,
-    dao_update_service,
     delete_service_and_all_associated_db_objects,
+    dao_fetch_all_services_by_user,
+    dao_fetch_service_by_id,
+    dao_update_service
 )
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.users_dao import (
-    delete_model_user,
-    delete_user_verify_codes,
-    get_user_by_email,
-)
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes, get_user_by_email
 from app.models import (
-    KEY_TYPE_TEST,
+    PROVIDERS,
     NOTIFICATION_CREATED,
+    KEY_TYPE_TEST,
     SMS_TYPE,
-    AnnualBilling,
-    Domain,
-    EmailBranding,
+    EMAIL_TYPE,
+    LETTER_TYPE,
+    User,
     Notification,
     Organisation,
+    Domain,
     Service,
-    Template,
-    TemplateHistory,
-    User,
+    EmailBranding,
+    LetterBranding,
 )
-from app.utils import get_local_midnight_in_utc
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before
 
 
 @click.group(name='command', help='Additional commands')
@@ -76,28 +72,32 @@ class notify_command:
         self.name = name
 
     def __call__(self, func):
-        decorators = [
-            functools.wraps(func),  # carry through function name, docstrings, etc.
-            click.command(name=self.name),  # turn it into a click.Command
-        ]
-
-        # in the test environment the app context is already provided and having
-        # another will lead to the test db connection being closed prematurely
-        if getenv('NOTIFY_ENVIRONMENT', '') != 'test':
-            # with_appcontext ensures the config is loaded, db connected, etc.
-            decorators.insert(0, flask.cli.with_appcontext)
-
+        # we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
+        # we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
+        # Then we need to turn it into a click.Command - that's what command_group.add_command expects.
+        @click.command(name=self.name)
+        @functools.wraps(func)
+        @flask.cli.with_appcontext
         def wrapper(*args, **kwargs):
             return func(*args, **kwargs)
 
-        for decorator in decorators:
-            # this syntax is equivalent to e.g. "@flask.cli.with_appcontext"
-            wrapper = decorator(wrapper)
-
         command_group.add_command(wrapper)
+
         return wrapper
 
 
+@notify_command()
+@click.option('-p', '--provider_name', required=True, type=click.Choice(PROVIDERS))
+@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals', type=float)
+@click.option('-d', '--valid_from', required=True, type=click_dt(format='%Y-%m-%dT%H:%M:%S'))
+def create_provider_rates(provider_name, cost, valid_from):
+    """
+    Backfill rates for a given provider
+    """
+    cost = Decimal(cost)
+    dao_create_provider_rates(provider_name, valid_from, cost)
+
+
 @notify_command()
 @click.option('-u', '--user_email_prefix', required=True, help="""
     Functional test user email prefix. eg "notify-test-preview"
@@ -106,12 +106,8 @@ def purge_functional_test_data(user_email_prefix):
     """
     Remove non-seeded functional test data
 
-    users, services, etc. Give an email prefix. Probably "notify-tests-preview".
+    users, services, etc. Give an email prefix. Probably "notify-test-preview".
     """
-    if getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
     users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
     for usr in users:
         # Make sure the full email includes a uuid in it
@@ -123,44 +119,343 @@ def purge_functional_test_data(user_email_prefix):
         else:
             services = dao_fetch_all_services_by_user(usr.id)
             if services:
-                print(f"Deleting user {usr.id} which is part of services")
                 for service in services:
                     delete_service_and_all_associated_db_objects(service)
             else:
-                services_created_by_this_user = dao_fetch_all_services_created_by_user(usr.id)
-                if services_created_by_this_user:
-                    # user is not part of any services but may still have been the one to create the service
-                    # sometimes things get in this state if the tests fail half way through
-                    # Remove the service they created (but are not a part of) so we can then remove the user
-                    print(f"Deleting services created by {usr.id}")
-                    for service in services_created_by_this_user:
-                        delete_service_and_all_associated_db_objects(service)
-
-                print(f"Deleting user {usr.id} which is not part of any services")
                 delete_user_verify_codes(usr)
                 delete_model_user(usr)
 
 
+@notify_command()
+def backfill_notification_statuses():
+    """
+    DEPRECATED. Populates notification_status.
+
+    This will be used to populate the new `Notification._status_fkey` with the old
+    `Notification._status_enum`
+    """
+    LIMIT = 250000
+    subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
+    result = db.session.execute(subq).fetchall()
+
+    while len(result) > 0:
+        db.session.execute(update)
+        print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
+        db.session.commit()
+        result = db.session.execute(subq).fetchall()
+
+
+@notify_command()
+def update_notification_international_flag():
+    """
+    DEPRECATED. Set notifications.international=false.
+    """
+    # 250,000 rows takes 30 seconds to update.
+    subq = "select id from notifications where international is null limit 250000"
+    update = "update notifications set international = False where id in ({})".format(subq)
+    result = db.session.execute(subq).fetchall()
+
+    while len(result) > 0:
+        db.session.execute(update)
+        print('commit 250000 updates at {}'.format(datetime.utcnow()))
+        db.session.commit()
+        result = db.session.execute(subq).fetchall()
+
+    # Now update notification_history
+    subq_history = "select id from notification_history where international is null limit 250000"
+    update_history = "update notification_history set international = False where id in ({})".format(subq_history)
+    result_history = db.session.execute(subq_history).fetchall()
+    while len(result_history) > 0:
+        db.session.execute(update_history)
+        print('commit 250000 updates at {}'.format(datetime.utcnow()))
+        db.session.commit()
+        result_history = db.session.execute(subq_history).fetchall()
+
+
+@notify_command()
+def fix_notification_statuses_not_in_sync():
+    """
+    DEPRECATED.
+    This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
+    became out of sync. See 979e90a.
+
+    Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
+    these values.
+    """
+    MAX = 10000
+
+    subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
+    update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
+    result = db.session.execute(subq).fetchall()
+
+    while len(result) > 0:
+        db.session.execute(update)
+        print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
+        db.session.commit()
+        result = db.session.execute(subq).fetchall()
+
+    subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
+        .format(MAX)
+    update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
+    result = db.session.execute(subq_hist).fetchall()
+
+    while len(result) > 0:
+        db.session.execute(update)
+        print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
+        db.session.commit()
+        result = db.session.execute(subq_hist).fetchall()
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+
+    WARNING: This does not overwrite existing data. You need to delete
+             the existing data or Performance Platform will double-count.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_processing_time(start_date, end_date):
+    """
+    Send historical processing time to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+        # because the tz conversion funcs talk about midnight, and the midnight before last,
+        # we want to pretend we're running this from the next morning, so add one.
+        process_date = start_date + timedelta(days=i + 1)
+
+        process_start_date = get_midnight_for_day_before(process_date)
+        process_end_date = get_london_midnight_in_utc(process_date)
+
+        print('Sending notification processing-time for {} - {}'.format(
+            process_start_date.isoformat(),
+            process_end_date.isoformat()
+        ))
+        send_processing_time_for_start_and_end(process_start_date, process_end_date)
+
+
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='list-routes')
+def list_routes():
+    """List URLs of all application routes."""
+    for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
+        print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
+
+
 @notify_command(name='insert-inbound-numbers')
 @click.option('-f', '--file_name', required=True,
-              help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""")
+              help="""Full path of the file to upload, file is a contains inbound numbers,
+              one number per line. The number must have the format of 07... not 447....""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
-    with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'sns', null, True, now(), null);"
+    file = open(file_name)
+    sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
 
-        for line in file:
-            line = line.strip()
-            if line:
-                print(line)
-                db.session.execute(sql.format(uuid.uuid4(), line))
-                db.session.commit()
+    for line in file:
+        print(line)
+        db.session.execute(sql.format(uuid.uuid4(), line.strip()))
+        db.session.commit()
+    file.close()
+
+
+@notify_command(name='replay-create-pdf-letters')
+@click.option('-n', '--notification_id', type=click.UUID, required=True,
+              help="Notification id of the letter that needs the create_letters_pdf task replayed")
+def replay_create_pdf_letters(notification_id):
+    print("Create task to create_letters_pdf for notification: {}".format(notification_id))
+    create_letters_pdf.apply_async([str(notification_id)], queue=QueueNames.CREATE_LETTERS_PDF)
+
+
+@notify_command(name='replay-service-callbacks')
+@click.option('-f', '--file_name', required=True,
+              help="""Full path of the file to upload, file is a contains client references of
+              notifications that need the status to be sent to the service.""")
+@click.option('-s', '--service_id', required=True,
+              help="""The service that the callbacks are for""")
+def replay_service_callbacks(file_name, service_id):
+    print("Start send service callbacks for service: ", service_id)
+    callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
+    if not callback_api:
+        print("Callback api was not found for service: {}".format(service_id))
+        return
+
+    errors = []
+    notifications = []
+    file = open(file_name)
+
+    for ref in file:
+        try:
+            notification = Notification.query.filter_by(client_reference=ref.strip()).one()
+            notifications.append(notification)
+        except NoResultFound:
+            errors.append("Reference: {} was not found in notifications.".format(ref))
+
+    for e in errors:
+        print(e)
+    if errors:
+        raise Exception("Some notifications for the given references were not found")
+
+    for n in notifications:
+        data = {
+            "notification_id": str(n.id),
+            "notification_client_reference": n.client_reference,
+            "notification_to": n.to,
+            "notification_status": n.status,
+            "notification_created_at": n.created_at.strftime(DATETIME_FORMAT),
+            "notification_updated_at": n.updated_at.strftime(DATETIME_FORMAT),
+            "notification_sent_at": n.sent_at.strftime(DATETIME_FORMAT),
+            "notification_type": n.notification_type,
+            "service_callback_api_url": callback_api.url,
+            "service_callback_api_bearer_token": callback_api.bearer_token,
+        }
+        encrypted_status_update = encryption.encrypt(data)
+        send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
+                                                    queue=QueueNames.CALLBACKS)
+
+    print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
+        service_id, len(notifications)))
 
 
 def setup_commands(application):
     application.cli.add_command(command_group)
 
 
+@notify_command(name='migrate-data-to-ft-billing')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@statsd(namespace="tasks")
+def migrate_data_to_ft_billing(start_date, end_date):
+
+    current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
+
+    process_date = start_date
+    total_updated = 0
+
+    while process_date < end_date:
+        start_time = datetime.utcnow()
+        # migrate data into ft_billing, upserting the data if it the record already exists
+        sql = \
+            """
+            insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
+                international, billable_units, notifications_sent, rate, postage, created_at)
+                select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
+                    case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
+                from (
+                    select
+                        n.id,
+                        (n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
+                        coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
+                        coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
+                        n.notification_type,
+                        coalesce(n.sent_by, (
+                        case
+                        when notification_type = 'sms' then
+                            coalesce(sent_by, 'unknown')
+                        when notification_type = 'letter' then
+                            coalesce(sent_by, 'dvla')
+                        else
+                            coalesce(sent_by, 'ses')
+                        end )) as provider,
+                        coalesce(n.rate_multiplier,1) as rate_multiplier,
+                        s.crown,
+                        coalesce((select rates.rate from rates
+                        where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
+                        order by rates.valid_from desc limit 1), 0) as sms_rate,
+                        coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
+                        and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
+                        and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
+                        as letter_rate,
+                        coalesce(n.international, false) as international,
+                        n.billable_units,
+                        1 as notifications_sent,
+                        coalesce(n.postage, 'none') as postage,
+                        now() as created_at
+                    from public.notification_history n
+                    left join services s on s.id = n.service_id
+                    where n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
+                        and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
+                        at time zone 'UTC'
+                        and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
+                    ) as individual_record
+                group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
+                    sms_rate, letter_rate, postage, created_at
+                order by bst_date
+            on conflict on constraint ft_billing_pkey do update set
+             billable_units = excluded.billable_units,
+             notifications_sent = excluded.notifications_sent,
+             rate = excluded.rate,
+             updated_at = now()
+            """
+
+        result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
+        db.session.commit()
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
+
+        process_date += timedelta(days=1)
+
+        total_updated += result.rowcount
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
 @notify_command(name='rebuild-ft-billing-for-day')
 @click.option('-s', '--service_id', required=False, type=click.UUID)
 @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -193,13 +488,36 @@ def rebuild_ft_billing_for_day(service_id, day):
         rebuild_ft_data(day, service_id)
     else:
         services = get_service_ids_that_need_billing_populated(
-            get_local_midnight_in_utc(day),
-            get_local_midnight_in_utc(day + timedelta(days=1))
+            get_london_midnight_in_utc(day),
+            get_london_midnight_in_utc(day + timedelta(days=1))
         )
         for row in services:
             rebuild_ft_data(day, row.service_id)
 
 
+@notify_command(name='migrate-data-to-ft-notification-status')
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
+@statsd(namespace="tasks")
+def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
+    notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
+
+    start_date = start_date.date()
+    end_date = end_date.date()
+    for day_diff in range((end_date - start_date).days + 1):
+        process_day = start_date + timedelta(days=day_diff)
+        for notification_type in notification_types:
+            print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
+                process_day,
+                notification_type
+            ))
+            create_nightly_notification_status_for_day.apply_async(
+                kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
+                queue=QueueNames.REPORTING
+            )
+
+
 @notify_command(name='bulk-invite-user-to-service')
 @click.option('-f', '--file_name', required=True,
               help="Full path of the file containing a list of email address for people to invite to a service")
@@ -211,12 +529,12 @@ def rebuild_ft_billing_for_day(service_id, day):
 def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permissions):
     #  permissions
     #  manage_users | manage_templates | manage_settings
-    #  send messages ==> send_texts | send_emails
+    #  send messages ==> send_texts | send_emails | send_letters
     #  Access API keys manage_api_keys
     #  platform_admin
     #  view_activity
-    # "send_texts,send_emails,view_activity"
-    from app.service_invite.rest import create_invited_user
+    # "send_texts,send_emails,send_letters,view_activity"
+    from app.invite.rest import create_invited_user
     file = open(file_name)
     for email_address in file:
         data = {
@@ -244,6 +562,52 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi
     file.close()
 
 
+@notify_command(name='populate-notification-postage')
+@click.option(
+    '-s',
+    '--start_date',
+    default=datetime(2017, 2, 1),
+    help="start date inclusive",
+    type=click_dt(format='%Y-%m-%d')
+)
+@statsd(namespace="tasks")
+def populate_notification_postage(start_date):
+    current_app.logger.info('populating historical notification postage')
+
+    total_updated = 0
+
+    while start_date < datetime.utcnow():
+        # process in ten day chunks
+        end_date = start_date + timedelta(days=10)
+
+        sql = \
+            """
+            UPDATE {}
+            SET postage = 'second'
+            WHERE notification_type = 'letter' AND
+            postage IS NULL AND
+            created_at BETWEEN :start AND :end
+            """
+
+        execution_start = datetime.utcnow()
+
+        if end_date > datetime.utcnow() - timedelta(days=8):
+            print('Updating notifications table as well')
+            db.session.execute(sql.format('notifications'), {'start': start_date, 'end': end_date})
+
+        result = db.session.execute(sql.format('notification_history'), {'start': start_date, 'end': end_date})
+        db.session.commit()
+
+        current_app.logger.info('notification postage took {}ms. Migrated {} rows for {} to {}'.format(
+            datetime.utcnow() - execution_start, result.rowcount, start_date, end_date))
+
+        start_date += timedelta(days=10)
+
+        total_updated += result.rowcount
+
+    current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
+
+
 @notify_command(name='archive-jobs-created-between-dates')
 @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
 @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@@ -259,9 +623,9 @@ def update_jobs_archived_flag(start_date, end_date):
         sql = """update
                     jobs set archived = true
                 where
-                    created_at >= (date :start + time '00:00:00') at time zone 'America/New_York'
+                    created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
                     at time zone 'UTC'
-                    and created_at < (date :end + time '00:00:00') at time zone 'America/New_York' at time zone 'UTC'"""
+                    and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'"""
 
         result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
         db.session.commit()
@@ -274,16 +638,56 @@ def update_jobs_archived_flag(start_date, end_date):
     current_app.logger.info('Total archived jobs = {}'.format(total_updated))
 
 
+@notify_command(name='update-emails-to-remove-gsi')
+@click.option('-s', '--service_id', required=True, help="service id. Update all user.email_address to remove .gsi")
+@statsd(namespace="tasks")
+def update_emails_to_remove_gsi(service_id):
+    users_to_update = """SELECT u.id user_id, u.name, email_address, s.id, s.name
+                           FROM users u
+                           JOIN user_to_service us on (u.id = us.user_id)
+                           JOIN services s on (s.id = us.service_id)
+                          WHERE s.id = :service_id
+                            AND u.email_address ilike ('%.gsi.gov.uk%')
+    """
+    results = db.session.execute(users_to_update, {'service_id': service_id})
+    print("Updating {} users.".format(results.rowcount))
+
+    for user in results:
+        print('User with id {} updated'.format(user.user_id))
+
+        update_stmt = """
+        UPDATE users
+           SET email_address = replace(replace(email_address, '.gsi.gov.uk', '.gov.uk'), '.GSI.GOV.UK', '.GOV.UK'),
+               updated_at = now()
+         WHERE id = :user_id
+        """
+        db.session.execute(update_stmt, {'user_id': str(user.user_id)})
+        db.session.commit()
+
+
+@notify_command(name='replay-daily-sorted-count-files')
+@click.option('-f', '--file_extension', required=False, help="File extension to search for, defaults to rs.txt")
+@statsd(namespace="tasks")
+def replay_daily_sorted_count_files(file_extension):
+    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
+    for filename in s3.get_list_of_files_by_suffix(bucket_name=bucket_location,
+                                                   subfolder='root/dispatch',
+                                                   suffix=file_extension or '.rs.txt'):
+        print("Create task to record daily sorted counts for file: ", filename)
+        record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
+
+
 @notify_command(name='populate-organisations-from-file')
 @click.option('-f', '--file_name', required=True,
               help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains")
 def populate_organisations_from_file(file_name):
     # [0] organisation name:: name of the organisation insert if organisation is missing.
-    # [1] sector:: Federal | State only
+    # [1] sector:: Central | Local | NHS only
     # [2] crown:: TRUE | FALSE only
     # [3] argeement_signed:: TRUE | FALSE
     # [4] domains:: comma separated list of domains related to the organisation
     # [5] email branding name: name of the default email branding for the org
+    # [6] letter branding name: name of the default letter branding for the org
 
     # The expectation is that the organisation, organisation_to_service
     # and user_to_organisation will be cleared before running this command.
@@ -304,13 +708,19 @@ def populate_organisations_from_file(file_name):
             email_branding_column = columns[5].strip()
             if len(email_branding_column) > 0:
                 email_branding = EmailBranding.query.filter(EmailBranding.name == email_branding_column).one()
+            letter_branding = None
+            letter_branding_column = columns[6].strip()
+            if len(letter_branding_column) > 0:
+                letter_branding = LetterBranding.query.filter(LetterBranding.name == letter_branding_column).one()
             data = {
                 'name': columns[0],
                 'active': True,
                 'agreement_signed': boolean_or_none(columns[3]),
                 'crown': boolean_or_none(columns[2]),
                 'organisation_type': columns[1].lower(),
-                'email_branding_id': email_branding.id if email_branding else None
+                'email_branding_id': email_branding.id if email_branding else None,
+                'letter_branding_id': letter_branding.id if letter_branding else None
+
             }
             org = Organisation(**data)
             try:
@@ -331,38 +741,40 @@ def populate_organisations_from_file(file_name):
                         db.session.rollback()
 
 
-@notify_command(name='populate-organisation-agreement-details-from-file')
-@click.option('-f', '--file_name', required=True,
-              help="CSV file containing id, agreement_signed_version, "
-              "agreement_signed_on_behalf_of_name, agreement_signed_at")
-def populate_organisation_agreement_details_from_file(file_name):
+@notify_command(name='get-letter-details-from-zips-sent-file')
+@click.argument('file_paths', required=True, nargs=-1)
+@statsd(namespace="tasks")
+def get_letter_details_from_zips_sent_file(file_paths):
+    """Get notification details from letters listed in zips_sent file(s)
+
+    This takes one or more file paths for the zips_sent files in S3 as its parameters, for example:
+    get-letter-details-from-zips-sent-file '2019-04-01/zips_sent/filename_1' '2019-04-01/zips_sent/filename_2'
     """
-    The input file should be a comma separated CSV file with a header row and 4 columns
-    id: the organisation ID
-    agreement_signed_version
-    agreement_signed_on_behalf_of_name
-    agreement_signed_at: The date the agreement was signed in the format of 'dd/mm/yyyy'
-    """
-    with open(file_name) as f:
-        csv_reader = csv.reader(f)
 
-        # ignore the header row
-        next(csv_reader)
+    rows_from_file = []
 
-        for row in csv_reader:
-            org = dao_get_organisation_by_id(row[0])
+    for path in file_paths:
+        file_contents = s3.get_s3_file(
+            bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+            file_location=path
+        )
+        rows_from_file.extend(json.loads(file_contents))
 
-            current_app.logger.info(f"Updating {org.name}")
+    notification_references = tuple(row[18:34] for row in rows_from_file)
 
-            if not org.agreement_signed:
-                raise RuntimeError('Agreement was not signed')
+    sql = """
+        SELECT id, service_id, reference, job_id, created_at
+        FROM notifications
+        WHERE reference IN :notification_references
+        ORDER BY service_id, job_id"""
+    result = db.session.execute(sql, {'notification_references': notification_references}).fetchall()
 
-            org.agreement_signed_version = float(row[1])
-            org.agreement_signed_on_behalf_of_name = row[2].strip()
-            org.agreement_signed_at = datetime.strptime(row[3], "%d/%m/%Y")
+    with open('zips_sent_details.csv', 'w') as csvfile:
+        csv_writer = csv.writer(csvfile)
+        csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at'])
 
-            db.session.add(org)
-            db.session.commit()
+        for row in result:
+            csv_writer.writerow(row)
 
 
 @notify_command(name='associate-services-to-organisations')
@@ -383,11 +795,12 @@ def associate_services_to_organisations():
 
 @notify_command(name='populate-service-volume-intentions')
 @click.option('-f', '--file_name', required=True,
-              help="Pipe delimited file containing service_id, SMS, email")
+              help="Pipe delimited file containing service_id, SMS, email, letters")
 def populate_service_volume_intentions(file_name):
     # [0] service_id
     # [1] SMS:: volume intentions for service
     # [2] Email:: volume intentions for service
+    # [3] Letters:: volume intentions for service
 
     with open(file_name, 'r') as f:
         for line in itertools.islice(f, 1, None):
@@ -396,6 +809,7 @@ def populate_service_volume_intentions(file_name):
             service = dao_fetch_service_by_id(columns[0])
             service.volume_sms = columns[1]
             service.volume_email = columns[2]
+            service.volume_letter = columns[3]
             dao_update_service(service)
     print("populate-service-volume-intentions complete")
 
@@ -475,7 +889,8 @@ def process_row_from_job(job_id, job_row_number):
     job = dao_get_job_by_id(job_id)
     db_template = dao_get_template_by_id(job.template_id, job.template_version)
 
-    template = db_template._as_utils_template()
+    TemplateClass = get_template_class(db_template.template_type)
+    template = TemplateClass(db_template.__dict__)
 
     for row in RecipientCSV(
             s3.get_job_from_s3(str(job.service_id), str(job.id)),
@@ -486,169 +901,3 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
-
-
-@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing_with_the_previous_years_allowance(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
-@notify_command(name='populate-annual-billing-with-defaults')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2021""")
-@click.option('-m', '--missing-services-only', default=True, type=bool,
-              help="""If true then only populate services missing from annual billing for the year.
-                      If false populate the default values for all active services.""")
-def populate_annual_billing_with_defaults(year, missing_services_only):
-    """
-    Add or update annual billing with free allowance defaults for all active services.
-    The default free allowance limits are in: app/dao/annual_billing_dao.py:57.
-
-    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
-    This is useful to prevent overriding any services that have a free allowance that is not the default.
-
-    If missing_services_only is false then add or update annual billing for all active services.
-    This is useful to ensure all services start the new year with the correct annual billing.
-    """
-    if missing_services_only:
-        active_services = Service.query.filter(
-            Service.active
-        ).outerjoin(
-            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
-        ).filter(
-            AnnualBilling.id == None  # noqa
-        ).all()
-    else:
-        active_services = Service.query.filter(
-            Service.active
-        ).all()
-    previous_year = year - 1
-    services_with_zero_free_allowance = db.session.query(AnnualBilling.service_id).filter(
-        AnnualBilling.financial_year_start == previous_year,
-        AnnualBilling.free_sms_fragment_limit == 0
-    ).all()
-
-    for service in active_services:
-
-        # If a service has free_sms_fragment_limit for the previous year
-        # set the free allowance for this year to 0 as well.
-        # Else use the default free allowance for the service.
-        if service.id in [x.service_id for x in services_with_zero_free_allowance]:
-            print(f'update service {service.id} to 0')
-            dao_create_or_update_annual_billing_for_year(
-                service_id=service.id,
-                free_sms_fragment_limit=0,
-                financial_year_start=year
-            )
-        else:
-            print(f'update service {service.id} with default')
-            set_default_free_allowance_for_service(service, year)
-
-
-def validate_mobile(ctx, param, value):
-    if (len(''.join(i for i in value if i.isdigit())) != 10):
-        raise click.BadParameter("mobile number must have 10 digits")
-    else:
-        return value
-
-
-@notify_command(name='create-test-user')
-@click.option('-n', '--name', required=True, prompt=True)
-@click.option('-e', '--email', required=True, prompt=True)  # TODO: require valid email
-@click.option('-m', '--mobile_number',
-              required=True, prompt=True, callback=validate_mobile)
-@click.option('-p', '--password',
-              required=True, prompt=True, hide_input=True, confirmation_prompt=True)
-@click.option('-a', '--auth_type', default="sms_auth")
-@click.option('-s', '--state', default="active")
-@click.option('-d', '--admin', default=False, type=bool)
-def create_test_user(name, email, mobile_number, password, auth_type, state, admin):
-    if getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']:
-        current_app.logger.error('Can only be run in development')
-        return
-
-    data = {
-        'name': name,
-        'email_address': email,
-        'mobile_number': mobile_number,
-        'password': password,
-        'auth_type': auth_type,
-        'state': state,  # skip the email verification for our test user
-        'platform_admin': admin,
-    }
-    user = User(**data)
-    try:
-        db.session.add(user)
-        db.session.commit()
-    except IntegrityError:
-        print("duplicate user", user.name)
-        db.session.rollback()
-
-
-@notify_command(name='create-admin-jwt')
-def create_admin_jwt():
-    if getenv('NOTIFY_ENVIRONMENT', '') != 'development':
-        current_app.logger.error('Can only be run in development')
-        return
-    print(create_jwt_token(current_app.config['SECRET_KEY'], current_app.config['ADMIN_CLIENT_ID']))
-
-
-@notify_command(name='create-user-jwt')
-@click.option('-t', '--token', required=True, prompt=False)
-def create_user_jwt(token):
-    if getenv('NOTIFY_ENVIRONMENT', '') != 'development':
-        current_app.logger.error('Can only be run in development')
-        return
-    service_id = token[-73:-37]
-    api_key = token[-36:]
-    print(create_jwt_token(api_key, service_id))
-
-
-def _update_template(id, name, template_type, content, subject):
-
-    template = Template.query.filter_by(id=id).first()
-    template.name = name
-    template.template_type = template_type
-    template.content = '\n'.join(content)
-    template.subject = subject
-
-    history = TemplateHistory.query.filter_by(id=id).first()
-    history.name = name
-    history.template_type = template_type
-    history.content = '\n'.join(content)
-    history.subject = subject
-
-    db.session.commit()
-
-
-@notify_command(name='update-templates')
-def update_templates():
-    with open(current_app.config['CONFIG_FILES'] + '/templates.json') as f:
-        data = json.load(f)
-        for d in data:
-            _update_template(d['id'], d['name'], d['type'], d['content'], d['subject'])

app/complaint/complaint_rest.py

@@ -3,10 +3,7 @@ from datetime import datetime
 from flask import Blueprint, jsonify, request
 
 from app.complaint.complaint_schema import complaint_count_request
-from app.dao.complaint_dao import (
-    fetch_count_of_complaints,
-    fetch_paginated_complaints,
-)
+from app.dao.complaint_dao import fetch_count_of_complaints, fetch_paginated_complaints
 from app.errors import register_errors
 from app.schema_validation import validate
 from app.utils import pagination_links

app/complaint/complaint_schema.py

@@ -1,6 +1,6 @@
 
 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "complaint count request schema",
     "type": "object",
     "title": "Complaint count request",

app/config.py

@@ -1,11 +1,16 @@
-import json
 from datetime import timedelta
-from os import getenv, path
+import os
+import json
 
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 
-from app.cloudfoundry_config import cloud_config
+if os.environ.get('VCAP_SERVICES'):
+    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
+    # standard environment variables from it
+    from app.cloudfoundry_config import extract_cloudfoundry_config
+
+    extract_cloudfoundry_config()
 
 
 class QueueNames(object):
@@ -19,12 +24,12 @@ class QueueNames(object):
     JOBS = 'job-tasks'
     RETRY = 'retry-tasks'
     NOTIFY = 'notify-internal-tasks'
+    PROCESS_FTP = 'process-ftp-tasks'
+    CREATE_LETTERS_PDF = 'create-letters-pdf-tasks'
     CALLBACKS = 'service-callbacks'
-    CALLBACKS_RETRY = 'service-callbacks-retry'
-    SMS_CALLBACKS = 'sms-callbacks'
+    LETTERS = 'letter-tasks'
     ANTIVIRUS = 'antivirus-tasks'
-    SAVE_API_EMAIL = 'save-api-email-tasks'
-    SAVE_API_SMS = 'save-api-sms-tasks'
+    SANITISE_LETTERS = 'sanitise-letter-tasks'
 
     @staticmethod
     def all_queues():
@@ -39,108 +44,112 @@ class QueueNames(object):
             QueueNames.JOBS,
             QueueNames.RETRY,
             QueueNames.NOTIFY,
+            QueueNames.CREATE_LETTERS_PDF,
             QueueNames.CALLBACKS,
-            QueueNames.CALLBACKS_RETRY,
-            QueueNames.SMS_CALLBACKS,
-            QueueNames.SAVE_API_EMAIL,
-            QueueNames.SAVE_API_SMS,
+            QueueNames.LETTERS,
         ]
 
 
 class TaskNames(object):
     PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
+    ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
     SCAN_FILE = 'scan-file'
+    SANITISE_LETTER = 'sanitise-and-upload-letter'
 
 
 class Config(object):
-    NOTIFY_APP_NAME = 'api'
-    NOTIFY_ENVIRONMENT = getenv('NOTIFY_ENVIRONMENT', 'development')
     # URL of admin app
-    ADMIN_BASE_URL = getenv('ADMIN_BASE_URL', 'http://localhost:6012')
+    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
+
     # URL of api app (on AWS this is the internal api endpoint)
-    API_HOST_NAME = getenv('API_HOST_NAME', 'http://localhost:6011')
+    API_HOST_NAME = os.getenv('API_HOST_NAME')
+
+    # admin app api key
+    ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET')
 
-    # Credentials
-    # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
-    ADMIN_CLIENT_ID = getenv('ADMIN_CLIENT_ID', 'notify-admin')
-    INTERNAL_CLIENT_API_KEYS = json.loads(
-        getenv(
-            'INTERNAL_CLIENT_API_KEYS',
-            ('{"%s":["%s"]}' % (ADMIN_CLIENT_ID, getenv('ADMIN_CLIENT_SECRET')))
-            )
-    )
-    ALLOW_EXPIRED_API_TOKEN = False
     # encyption secret/salt
-    SECRET_KEY = getenv('SECRET_KEY')
-    DANGEROUS_SALT = getenv('DANGEROUS_SALT')
-    ROUTE_SECRET_KEY_1 = getenv('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
-    ROUTE_SECRET_KEY_2 = getenv('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
+    SECRET_KEY = os.getenv('SECRET_KEY')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
 
-    # DB settings
-    SQLALCHEMY_DATABASE_URI = cloud_config.database_url
+    # DB conection string
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
+
+    # MMG API Key
+    MMG_API_KEY = os.getenv('MMG_API_KEY')
+
+    # Firetext API Key
+    FIRETEXT_API_KEY = os.getenv("FIRETEXT_API_KEY")
+
+    # Hosted graphite statsd prefix
+    STATSD_PREFIX = os.getenv('STATSD_PREFIX')
+
+    # Prefix to identify queues in SQS
+    NOTIFICATION_QUEUE_PREFIX = os.getenv('NOTIFICATION_QUEUE_PREFIX')
+
+    # URL of redis instance
+    REDIS_URL = os.getenv('REDIS_URL')
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
+    EXPIRE_CACHE_TEN_MINUTES = 600
+    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
+
+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = False
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
+
+    # Zendesk
+    ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY')
+
+    # Logging
+    DEBUG = False
+    NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')
+
+    # Cronitor
+    CRONITOR_ENABLED = False
+    CRONITOR_KEYS = json.loads(os.environ.get('CRONITOR_KEYS', '{}'))
+
+    # Antivirus
+    ANTIVIRUS_ENABLED = True
+
+    ###########################
+    # Default config values ###
+    ###########################
+
+    NOTIFY_ENVIRONMENT = 'development'
+    ADMIN_CLIENT_USER_NAME = 'notify-admin'
+    AWS_REGION = 'eu-west-1'
+    INVITATION_EXPIRATION_DAYS = 2
+    NOTIFY_APP_NAME = 'api'
     SQLALCHEMY_RECORD_QUERIES = False
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-    SQLALCHEMY_POOL_SIZE = int(getenv('SQLALCHEMY_POOL_SIZE', 5))
+    SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5))
     SQLALCHEMY_POOL_TIMEOUT = 30
     SQLALCHEMY_POOL_RECYCLE = 300
     SQLALCHEMY_STATEMENT_TIMEOUT = 1200
     PAGE_SIZE = 50
     API_PAGE_SIZE = 250
-    REDIS_URL = cloud_config.redis_url
-    REDIS_ENABLED = getenv('REDIS_ENABLED', '0') == '1'
-    EXPIRE_CACHE_TEN_MINUTES = 600
-    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
+    TEST_MESSAGE_FILENAME = 'Test message'
+    ONE_OFF_MESSAGE_FILENAME = 'Report'
+    MAX_VERIFY_CODE_COUNT = 10
 
-    # AWS Settings
-    AWS_US_TOLL_FREE_NUMBER = getenv("AWS_US_TOLL_FREE_NUMBER")
-    # Whether to ignore POSTs from SNS for replies to SMS we sent
-    RECEIVE_INBOUND_SMS = False
-    NOTIFY_EMAIL_DOMAIN = cloud_config.ses_email_domain
-    SES_STUB_URL = None  # TODO: set to a URL in env and remove this to use a stubbed SES service
-    # AWS SNS topics for delivery receipts
-    VALIDATE_SNS_TOPICS = True
-    VALID_SNS_TOPICS = cloud_config.sns_topic_arns
+    # be careful increasing this size without being sure that we won't see slowness in pysftp
+    MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
+    MAX_LETTER_PDF_COUNT_PER_ZIP = 500
+
+    CHECK_PROXY_HEADER = False
 
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {
-        'sns': 100,
+        'mmg': 60,
+        'firetext': 40
     }
 
-    # Zendesk
-    ZENDESK_API_KEY = getenv('ZENDESK_API_KEY')
-
-    # Logging
-    DEBUG = False
-
-    # Monitoring
-    CRONITOR_ENABLED = False
-    CRONITOR_KEYS = json.loads(getenv('CRONITOR_KEYS', '{}'))
-    STATSD_HOST = getenv('STATSD_HOST')
-    STATSD_PORT = 8125
-    STATSD_ENABLED = bool(STATSD_HOST)
-
-    # Antivirus
-    ANTIVIRUS_ENABLED = getenv('ANTIVIRUS_ENABLED', '1') == '1'
-
-    SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200  # 3 days
-    INVITATION_EXPIRATION_DAYS = 2
-    TEST_MESSAGE_FILENAME = 'Test message'
-    ONE_OFF_MESSAGE_FILENAME = 'Report'
-    MAX_VERIFY_CODE_COUNT = 5
-    MAX_FAILED_LOGIN_COUNT = 10
-    API_RATE_LIMIT_ENABLED = True
-
-    # Default data
-    CONFIG_FILES = path.dirname(__file__) + '/config_files/'
-
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
     NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
     INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
     SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246'
     EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164'
     NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27'
-    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'  # nosec B105 - this is not a password
+    PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
     ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
     CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
     SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
@@ -151,208 +160,356 @@ class Config(object):
     MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
     MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = getenv('AWS_US_TOLL_FREE_NUMBER')
-    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
-    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
-    # we only need real email in Live environment (production)
-    DVLA_EMAIL_ADDRESSES = json.loads(getenv('DVLA_EMAIL_ADDRESSES', '[]'))
+    MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = 'd0e66c4c-0c50-43f0-94f5-f85b613202d4'
 
-    CELERY = {
-        'broker_url': REDIS_URL,
-        'broker_transport_options': {
-            'visibility_timeout': 310,
-        },
-        'timezone': getenv("TIMEZONE", 'America/New_York'),
-        'imports': [
-            'app.celery.tasks',
-            'app.celery.scheduled_tasks',
-            'app.celery.reporting_tasks',
-            'app.celery.nightly_tasks',
-        ],
-        # this is overriden by the -Q command, but locally, we should read from all queues
-        'task_queues': [
-            Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()
-        ],
-        'beat_schedule': {
-            # app/celery/scheduled_tasks.py
-            'run-scheduled-jobs': {
-                'task': 'run-scheduled-jobs',
-                'schedule': crontab(minute='0,15,30,45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-verify-codes': {
-                'task': 'delete-verify-codes',
-                'schedule': timedelta(minutes=63),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'delete-invitations': {
-                'task': 'delete-invitations',
-                'schedule': timedelta(minutes=66),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-job-status': {
-                'task': 'check-job-status',
-                'schedule': crontab(),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'tend-providers-back-to-middle': {
-                'task': 'tend-providers-back-to-middle',
-                'schedule': crontab(minute='*/5'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'check-for-missing-rows-in-completed-jobs': {
-                'task': 'check-for-missing-rows-in-completed-jobs',
-                'schedule': crontab(minute='*/10'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'replay-created-notifications': {
-                'task': 'replay-created-notifications',
-                'schedule': crontab(minute='0, 15, 30, 45'),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            # app/celery/nightly_tasks.py
-            'timeout-sending-notifications': {
-                'task': 'timeout-sending-notifications',
-                'schedule': crontab(hour=0, minute=5),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'create-nightly-billing': {
-                'task': 'create-nightly-billing',
-                'schedule': crontab(hour=0, minute=15),
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'create-nightly-notification-status': {
-                'task': 'create-nightly-notification-status',
-                'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-notifications-older-than-retention': {
-                'task': 'delete-notifications-older-than-retention',
-                'schedule': crontab(hour=3, minute=0),  # after 'create-nightly-notification-status'
-                'options': {'queue': QueueNames.REPORTING}
-            },
-            'delete-inbound-sms': {
-                'task': 'delete-inbound-sms',
-                'schedule': crontab(hour=1, minute=40),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'save-daily-notification-processing-time': {
-                'task': 'save-daily-notification-processing-time',
-                'schedule': crontab(hour=2, minute=0),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-            'remove_sms_email_jobs': {
-                'task': 'remove_sms_email_jobs',
-                'schedule': crontab(hour=4, minute=0),
-                'options': {'queue': QueueNames.PERIODIC},
-            },
-            'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
-                'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
-                'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
-                'options': {'queue': QueueNames.PERIODIC}
-            },
-        }
+    BROKER_URL = 'sqs://'
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 310,
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
     }
+    CELERY_ENABLE_UTC = True
+    CELERY_TIMEZONE = 'Europe/London'
+    CELERY_ACCEPT_CONTENT = ['json']
+    CELERY_TASK_SERIALIZER = 'json'
+    CELERY_IMPORTS = (
+        'app.celery.tasks',
+        'app.celery.scheduled_tasks',
+        'app.celery.reporting_tasks',
+        'app.celery.nightly_tasks',
+    )
+    CELERYBEAT_SCHEDULE = {
+        # app/celery/scheduled_tasks.py
+        'run-scheduled-jobs': {
+            'task': 'run-scheduled-jobs',
+            'schedule': crontab(minute=1),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-verify-codes': {
+            'task': 'delete-verify-codes',
+            'schedule': timedelta(minutes=63),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-invitations': {
+            'task': 'delete-invitations',
+            'schedule': timedelta(minutes=66),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-job-status': {
+            'task': 'check-job-status',
+            'schedule': crontab(),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'tend-providers-back-to-middle': {
+            'task': 'tend-providers-back-to-middle',
+            'schedule': crontab(minute='*/5'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-missing-rows-in-completed-jobs': {
+            'task': 'check-for-missing-rows-in-completed-jobs',
+            'schedule': crontab(minute='*/10'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'replay-created-notifications': {
+            'task': 'replay-created-notifications',
+            'schedule': crontab(minute='0, 15, 30, 45'),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # app/celery/nightly_tasks.py
+        'timeout-sending-notifications': {
+            'task': 'timeout-sending-notifications',
+            'schedule': crontab(hour=0, minute=5),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=0, minute=15),
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'create-nightly-notification-status': {
+            'task': 'create-nightly-notification-status',
+            'schedule': crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            'options': {'queue': QueueNames.REPORTING}
+        },
+        'delete-sms-notifications': {
+            'task': 'delete-sms-notifications',
+            'schedule': crontab(hour=4, minute=15),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-email-notifications': {
+            'task': 'delete-email-notifications',
+            'schedule': crontab(hour=4, minute=30),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-letter-notifications': {
+            'task': 'delete-letter-notifications',
+            'schedule': crontab(hour=4, minute=45),  # after 'create-nightly-notification-status'
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'delete-inbound-sms': {
+            'task': 'delete-inbound-sms',
+            'schedule': crontab(hour=1, minute=40),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
 
-    # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
-    if getenv('CELERYD_PREFETCH_MULTIPLIER'):
-        CELERY['worker_prefetch_multiplier'] = getenv('CELERYD_PREFETCH_MULTIPLIER')
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'remove_sms_email_jobs': {
+            'task': 'remove_sms_email_jobs',
+            'schedule': crontab(hour=4, minute=0),
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'remove_letter_jobs': {
+            'task': 'remove_letter_jobs',
+            'schedule': crontab(hour=4, minute=20),
+            # since we mark jobs as archived
+            'options': {'queue': QueueNames.PERIODIC},
+        },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
+            'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
+            'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-letter-notifications-still-sending': {
+            'task': 'raise-alert-if-letter-notifications-still-sending',
+            'schedule': crontab(hour=15, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-for-day': {
+            'task': 'collate-letter-pdfs-for-day',
+            'schedule': crontab(hour=17, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+        'raise-alert-if-no-letter-ack-file': {
+            'task': 'raise-alert-if-no-letter-ack-file',
+            'schedule': crontab(hour=23, minute=00),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
+    }
+    CELERY_QUEUES = []
 
     FROM_NUMBER = 'development'
 
+    STATSD_HOST = os.getenv('STATSD_HOST')
+    STATSD_PORT = 8125
+    STATSD_ENABLED = bool(STATSD_HOST)
+
+    SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200  # 3 days
+
     SIMULATED_EMAIL_ADDRESSES = (
         'simulate-delivered@notifications.service.gov.uk',
         'simulate-delivered-2@notifications.service.gov.uk',
         'simulate-delivered-3@notifications.service.gov.uk',
     )
-    SIMULATED_SMS_NUMBERS = ('+12028675000', '+12028675111', '+12028675222')
+
+    SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
-    DAILY_MESSAGE_LIMIT = 5000
+    SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
+    FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
+    MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
+    MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
 
-    HIGH_VOLUME_SERVICE = json.loads(getenv('HIGH_VOLUME_SERVICE', '[]'))
+    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
+    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
 
-    TEMPLATE_PREVIEW_API_HOST = getenv('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
-    TEMPLATE_PREVIEW_API_KEY = getenv('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
+    # Format is as follows:
+    # {"dataset_1": "token_1", ...}
+    PERFORMANCE_PLATFORM_ENDPOINTS = json.loads(os.environ.get('PERFORMANCE_PLATFORM_ENDPOINTS', '{}'))
 
-    DOCUMENT_DOWNLOAD_API_HOST = getenv('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000')
-    DOCUMENT_DOWNLOAD_API_KEY = getenv('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
+    TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
+    TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
+
+    DOCUMENT_DOWNLOAD_API_HOST = os.environ.get('DOCUMENT_DOWNLOAD_API_HOST', 'http://localhost:7000')
+    DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
+
+    # these environment vars aren't defined in the manifest so to set them on paas use `cf set-env`
+    MMG_URL = os.environ.get("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
+    FIRETEXT_URL = os.environ.get("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
+
+    AWS_REGION = 'eu-west-1'
 
 
-def _s3_credentials_from_env(bucket_prefix):
-    return {
-        'bucket': getenv(f"{bucket_prefix}_BUCKET_NAME"),
-        'access_key_id': getenv(f"{bucket_prefix}_AWS_ACCESS_KEY_ID"),
-        'secret_access_key': getenv(f"{bucket_prefix}_AWS_SECRET_ACCESS_KEY"),
-        'region': getenv(f"{bucket_prefix}_AWS_REGION")
-    }
-
+######################
+# Config overrides ###
+######################
 
 class Development(Config):
     DEBUG = True
     SQLALCHEMY_ECHO = False
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
 
-    # Buckets
-    CSV_UPLOAD_BUCKET = _s3_credentials_from_env('CSV')
-    CONTACT_LIST_BUCKET = _s3_credentials_from_env('CONTACT')
+    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'development-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'development-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
 
-    # credential overrides
-    DANGEROUS_SALT = 'development-notify-salt'
-    SECRET_KEY = 'dev-notify-secret-key'  # nosec B105 - this is only used in development
-    INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
-    ALLOW_EXPIRED_API_TOKEN = getenv('ALLOW_EXPIRED_API_TOKEN', '0') == '1'
+    ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
+    SECRET_KEY = 'dev-notify-secret-key'
+    DANGEROUS_SALT = 'dev-notify-salt'
+
+    MMG_INBOUND_SMS_AUTH = ['testkey']
+    MMG_INBOUND_SMS_USERNAME = ['username']
+
+    NOTIFY_ENVIRONMENT = 'development'
+    NOTIFY_LOG_PATH = 'application.log'
+    NOTIFICATION_QUEUE_PREFIX = 'development'
+    NOTIFY_EMAIL_DOMAIN = "notify.tools"
+
+    SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
+    REDIS_URL = 'redis://localhost:6379/0'
+
+    ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
+
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
+    API_HOST_NAME = "http://localhost:6011"
+    API_RATE_LIMIT_ENABLED = True
 
 
 class Test(Development):
+    NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
     FROM_NUMBER = 'testing'
+    NOTIFY_ENVIRONMENT = 'test'
     TESTING = True
+
+    CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'test-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'test-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise'
+
+    # this is overriden in jenkins and on cloudfoundry
+    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/test_notification_api')
+
+    BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
+
     ANTIVIRUS_ENABLED = True
-    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
 
-    HIGH_VOLUME_SERVICE = [
-        '941b6f9a-50d7-4742-8d50-f365ca74bf27',
-        '63f95b86-2d19-4497-b8b2-ccf25457df4e',
-        '7e5950cb-9954-41f5-8376-962b8c8555cf',
-        '10d1b9c9-0072-4fa9-ae1c-595e333841da',
-    ]
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
 
-    # this is overriden in CI
-    SQLALCHEMY_DATABASE_URI = getenv('SQLALCHEMY_DATABASE_TEST_URI')
-
-    CELERY = {
-        **Config.CELERY,
-        'broker_url': 'you-forgot-to-mock-celery-in-your-tests://'
-    }
+    API_RATE_LIMIT_ENABLED = True
+    API_HOST_NAME = "http://localhost:6011"
 
+    SMS_INBOUND_WHITELIST = ['203.0.113.195']
+    FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
     TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
 
+    MMG_URL = 'https://example.com/mmg'
+    FIRETEXT_URL = 'https://example.com/firetext'
 
-class Production(Config):
-    # buckets
-    CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
-        f"notify-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}")
-    CONTACT_LIST_BUCKET = cloud_config.s3_credentials(
-        f"notify-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}")
 
-    FROM_NUMBER = 'US Notify'
+class Preview(Config):
+    NOTIFY_EMAIL_DOMAIN = 'notify.works'
+    NOTIFY_ENVIRONMENT = 'preview'
+    CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'preview-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'preview-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'preview-letters-sanitise'
+    FROM_NUMBER = 'preview'
+    API_RATE_LIMIT_ENABLED = True
+    CHECK_PROXY_HEADER = False
+
+
+class Staging(Config):
+    NOTIFY_EMAIL_DOMAIN = 'staging-notify.works'
+    NOTIFY_ENVIRONMENT = 'staging'
+    CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'staging-letters-sanitise'
+    FROM_NUMBER = 'stage'
+    API_RATE_LIMIT_ENABLED = True
+    CHECK_PROXY_HEADER = True
+    REDIS_ENABLED = True
+
+
+class Live(Config):
+    NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
+    NOTIFY_ENVIRONMENT = 'live'
+    CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
+    LETTER_SANITISE_BUCKET_NAME = 'production-letters-sanitise'
+    FROM_NUMBER = 'GOVUK'
+    PERFORMANCE_PLATFORM_ENABLED = True
+    API_RATE_LIMIT_ENABLED = True
+    CHECK_PROXY_HEADER = True
+
     CRONITOR_ENABLED = True
 
 
-class Staging(Production):
+class CloudFoundryConfig(Config):
     pass
 
 
-class Demo(Production):
-    pass
+# CloudFoundry sandbox
+class Sandbox(CloudFoundryConfig):
+    NOTIFY_EMAIL_DOMAIN = 'notify.works'
+    NOTIFY_ENVIRONMENT = 'sandbox'
+    CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
+    FROM_NUMBER = 'sandbox'
+    REDIS_ENABLED = False
 
 
 configs = {
     'development': Development,
     'test': Test,
+    'live': Live,
+    'production': Live,
     'staging': Staging,
-    'demo': Demo,
-    'sandbox': Staging,
-    'production': Production
+    'preview': Preview,
+    'sandbox': Sandbox
 }

app/config_files/templates.json

@@ -1,274 +0,0 @@
-[
-    {
-        "id": "42a23d19-504e-49bb-a95e-4976baff4757",
-        "name": "Example text message template",
-        "type": "sms",
-        "subject":  "",
-        "content": ["Hi, I’m trying out U.S. Notify. Today is ((day of week)) and my favorite color is ((color))."]
-    },
-    {
-        "id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
-        "name": "Notify invitation email",
-        "type": "email",
-        "subject":  "((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify",
-        "content": ["((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify.",
-            "",
-            "",
-            "U.S. Notify makes it easy to keep people updated by helping you send text messages and emails.",
-            "",
-            "",
-            "Click this link to create an account on U.S. Notify:",
-            "",
-            "((url))",
-            "",
-            "",
-            "This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure."]
-    },
-    {
-        "id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
-        "name": "Notify SMS verify code",
-        "type": "sms",
-        "subject": "",
-        "content": ["((verify_code)) is your U.S. Notify authentication code"]
-    },
-    {
-        "id": "474e9242-823b-4f99-813d-ed392e7f1201",
-        "name": "Notify password reset email",
-        "type": "email",
-        "subject": "Reset your U.S. Notify password",
-        "content": ["Hi ((user_name)),",
-            "",
-            "",
-            "We received a request to reset your password on U.S. Notify.",
-            "",
-            "",
-            "If you didn’t request this email, you can ignore it – your password has not been changed.",
-            "",
-            "",
-            "To reset your password, click this link:","","","((url))"]
-    },
-    {
-        "id": "299726d2-dba6-42b8-8209-30e1d66ea164",
-        "name": "Notify email verify code",
-        "type": "email",
-        "subject": "Sign in to U.S. Notify",
-        "content": ["Hi ((name)),",
-        "",
-        "",
-        "To sign in to U.S. Notify please open this link:",
-        "",
-        "",
-        "((url))"]
-    },
-    {
-        "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
-        "name": "Notify email verification code",
-        "type": "email",
-        "subject": "Confirm U.S. Notify registration",
-        "content": ["Hi ((name)),",
-        "",
-        "",
-        "To complete your registration for U.S. Notify please click the link below",
-        "",
-        "",
-        "((url))"]
-    },
-    {
-        "id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
-        "name": "Your U.S. Notify account",
-        "type": "email",
-        "subject": "Your U.S. Notify account",
-        "content": ["You already have a U.S. Notify account with this email address.",
-        "",
-        "",
-        "Sign in here: ((signin_url))",
-        "",
-        "",
-        "If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
-        "",
-        "",
-        "",
-        "",
-        "If you didn’t try to register for a U.S. Notify account recently, please let us know here: ((feedback_url))"]
-    },
-    {
-        "id": "eb4d9930-87ab-4aef-9bce-786762687884",
-        "name": "Confirm new email address",
-        "type": "email",
-        "subject": "Confirm your email address for U.S. Notify",
-        "content": ["Hi ((name)),","","","Click this link to confirm your new email address:",
-            "",
-            "",
-            "((url))",
-            "",
-            "",
-            "If you didn’t try to change the email address for your U.S. Notify account, let us know here:",
-            "",
-            "",
-            "((feedback_url))"]
-    },
-    {
-        "id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
-        "name": "Automated \"You’re now live\" message",
-        "type": "email",
-        "subject": "((service name)) is now live on U.S. Notify",
-        "content": ["Hi ((name)),",
-        "",
-        "",
-        "((service name)) is now live on U.S. Notify.",
-        "",
-        "",
-        "You can send up to ((message limit)) messages per day.",
-        "",
-        "",
-        "As a live service, you’ll need to know who to contact if you have a question, or something goes wrong.",
-        "",
-        "",
-        "If our system status page shows a problem, then we’ve been alerted and are working on it – you don’t need to contact us.",
-        "",
-        "",
-        "#Problems or questions out of hours",
-        "",
-        "",
-        "We offer out of hours support for emergencies.",
-        "",
-        "",
-        "It’s only an emergency if:",
-        "",
-        "*   no one in your team can log in",
-        "",
-        "*   a ‘technical difficulties’ error appears when you try to upload a file",
-        "",
-        "*   a 500 response code appears when you try to send messages using the API",
-        "",
-        "",
-        "If you have one of these emergencies, email details to:",
-        "",
-        "notify-support@gsa.gov",
-        "",
-        "",
-        "^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.",
-        "",
-        "",
-        "We’ll get back to you within 30 minutes and give you hourly updates until the problem’s fixed.",
-        "",
-        "",
-        "For non-emergency problems or questions, use our support page and we’ll reply in office hours.",
-        "",
-        "",
-        "Thanks",
-        "",
-        "U.S. Notify team"]
-    },
-    {
-        "id": "203566f0-d835-47c5-aa06-932439c86573",
-        "name": "Notify organization invitation email",
-        "type": "email",
-        "subject": "((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify",
-        "content": ["((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify.","","","U.S. Notify makes it easy to keep people updated by helping you send text messages and emails.","","","Open this link to create an account on U.S. Notify:","","((url))","","","This invitation will stop working at midnight tomorrow. This is to keep ((organisation_name)) secure."]
-    },
-    {
-        "id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
-        "name": "Email address changed by service manager",
-        "type": "email",
-        "subject": "Your U.S. Notify email address has changed",
-        "content":  ["Dear ((name)),","","","((servicemanagername)) changed your Notify account email address to:","","","((email address))","","","You’ll need to use this email address next time you sign in.","","","Thanks","","","U.S. Notify team"]
-    },
-    {
-        "id": "8a31520f-4751-4789-8ea1-fe54496725eb",
-        "name": "Phone number changed by service manager",
-        "type": "sms",
-        "subject": "",
-        "content": ["Your mobile number was changed by ((servicemanagername)). Next time you sign in, your U.S. Notify authentication code will be sent to this phone."]
-    },
-    {
-        "id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
-        "name": "Verify email reply-to address for a service",
-        "type": "email",
-        "subject": "Your U.S. Notify reply-to email address",
-        "content": ["Hi,","","","This address has been provided as a reply-to email address for a U.S. Notify account.","","Any replies from users to emails they receive through U.S. Notify will come back to this email address.","","","This is just a quick check to make sure the address is valid.","","","No need to reply.","","","Thanks","","","U.S. Notify team"]
-    },
-    {
-        "id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
-        "name": "MOU Signed By Receipt",
-        "type": "email",
-        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
-        "content": [
-            "Hi ((signed_by_name)),",
-            "",
-            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. ",
-            "",
-            "If you need another copy of the agreement you can download it here: ((mou_link))",
-            "",
-            "",
-            "Thanks,",
-            "U.S. Notify team"
-        ]
-    },
-    {
-        "id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
-        "name": "MOU Signed On Behalf Of Receipt - Signed by",
-        "type": "email",
-        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
-        "content": [
-            "Hi ((signed_by_name)),",
-            "",
-            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
-            "",
-            "If you need another copy of the agreement you can download it here: ((mou_link))",
-            "",
-            "",
-            "Thanks,",
-            "U.S. Notify team"
-        ]
-    },
-    {
-        "id": "522b6657-5ca5-4368-a294-6b527703bd0b",
-        "name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
-        "type": "email",
-        "subject": "((org_name)) has accepted the U.S. Notify data sharing and financial agreement",
-        "content": [
-            "Hi ((on_behalf_of_name)),",
-            "",
-            "((signed_by_name)) has accepted the U.S. Notify data sharing and financial agreement on your behalf, for ((org_name)).",
-            "",
-            "U.S. Notify lets teams in the public sector send text messages and emails. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
-            "",
-            "If you need another copy of the agreement you can download it here: ((mou_link))",
-            "",
-            "",
-            "Thanks,",
-            "U.S. Notify team"
-        ]
-    },
-    {
-        "id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
-        "name": "MOU Signed Notify Team Alert",
-        "type": "email",
-        "subject": "Someone signed an MOU for an org on Notify",
-        "content": [
-            "What’s up Notifiers,",
-            "",
-            "((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
-            "",
-            "See how ((org_name)) is using Notify here: ((org_dashboard_link))"
-        ]
-    },
-    {
-        "id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
-        "name": "Notify daily letter volumes",
-        "type": "email",
-        "subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
-        "content": [
-            "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
-            "",
-            "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
-            "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
-            "((international_volume)) international letters (((international_sheets)) sheets).",
-            "",
-            "Thanks",
-            "",
-            "U.S. Notify team"
-        ]
-    }
-]

app/cronitor.py

@@ -1,19 +1,15 @@
-from functools import wraps
-
 import requests
+from functools import wraps
 from flask import current_app
 
 
 def cronitor(task_name):
+    # check if task_name is in config
     def decorator(func):
         def ping_cronitor(command):
             if not current_app.config['CRONITOR_ENABLED']:
                 return
 
-            # it's useful to have a log that a periodic task has started in case it
-            # get stuck without generating any other logs - we know it got this far
-            current_app.logger.info(f'Pinging Cronitor for Celery task {task_name}')
-
             task_slug = current_app.config['CRONITOR_KEYS'].get(task_name)
             if not task_slug:
                 current_app.logger.error(
@@ -42,11 +38,13 @@ def cronitor(task_name):
         @wraps(func)
         def inner_decorator(*args, **kwargs):
             ping_cronitor('run')
-            status = 'fail'
             try:
                 ret = func(*args, **kwargs)
                 status = 'complete'
                 return ret
+            except Exception:
+                status = 'fail'
+                raise
             finally:
                 ping_cronitor(status)
 

app/dao/__init__.py

@@ -1,5 +1,4 @@
 from sqlalchemy.exc import SQLAlchemyError
-
 from app import db
 
 

app/dao/annual_billing_dao.py

@@ -1,12 +1,10 @@
-from flask import current_app
-
 from app import db
-from app.dao.dao_utils import autocommit
-from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.dao_utils import transactional
 from app.models import AnnualBilling
+from app.dao.date_util import get_current_financial_year_start_year
 
 
-@autocommit
+@transactional
 def dao_create_or_update_annual_billing_for_year(service_id, free_sms_fragment_limit, financial_year_start):
     result = dao_get_free_sms_fragment_limit_for_year(service_id, financial_year_start)
 
@@ -25,7 +23,7 @@ def dao_get_annual_billing(service_id):
     ).order_by(AnnualBilling.financial_year_start).all()
 
 
-@autocommit
+@transactional
 def dao_update_annual_billing_for_future_years(service_id, free_sms_fragment_limit, financial_year_start):
     AnnualBilling.query.filter(
         AnnualBilling.service_id == service_id,
@@ -51,42 +49,3 @@ def dao_get_all_free_sms_fragment_limit(service_id):
     return AnnualBilling.query.filter_by(
         service_id=service_id,
     ).order_by(AnnualBilling.financial_year_start).all()
-
-
-def set_default_free_allowance_for_service(service, year_start=None):
-    default_free_sms_fragment_limits = {
-        'federal': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'state': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        },
-        'other': {
-            2020: 250_000,
-            2021: 150_000,
-            2022: 40_000,
-        }
-    }
-    if not year_start:
-        year_start = get_current_financial_year_start_year()
-    # handle cases where the year is less than 2020 or greater than 2021
-    if year_start < 2020:
-        year_start = 2020
-    if year_start > 2022:
-        year_start = 2022
-    if service.organisation_type:
-        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
-    else:
-        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
-                                f"{default_free_sms_fragment_limits['other'][year_start]}")
-        free_allowance = default_free_sms_fragment_limits['other'][year_start]
-
-    return dao_create_or_update_annual_billing_for_year(
-        service.id,
-        free_allowance,
-        year_start
-    )

app/dao/api_key_dao.py

@@ -1,14 +1,18 @@
 import uuid
 from datetime import datetime, timedelta
 
-from sqlalchemy import func, or_
-
 from app import db
-from app.dao.dao_utils import autocommit, version_class
 from app.models import ApiKey
 
+from app.dao.dao_utils import (
+    transactional,
+    version_class
+)
 
-@autocommit
+from sqlalchemy import or_, func
+
+
+@transactional
 @version_class(ApiKey)
 def save_model_api_key(api_key):
     if not api_key.id:
@@ -17,7 +21,7 @@ def save_model_api_key(api_key):
     db.session.add(api_key)
 
 
-@autocommit
+@transactional
 @version_class(ApiKey)
 def expire_api_key(service_id, api_key_id):
     api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()

app/dao/complaint_dao.py

@@ -4,12 +4,12 @@ from flask import current_app
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import Complaint
-from app.utils import get_local_midnight_in_utc
+from app.utils import get_london_midnight_in_utc
 
 
-@autocommit
+@transactional
 def save_complaint(complaint):
     db.session.add(complaint)
 
@@ -28,7 +28,7 @@ def fetch_complaints_by_service(service_id):
 
 
 def fetch_count_of_complaints(start_date, end_date):
-    start_date = get_local_midnight_in_utc(start_date)
-    end_date = get_local_midnight_in_utc(end_date + timedelta(days=1))
+    start_date = get_london_midnight_in_utc(start_date)
+    end_date = get_london_midnight_in_utc(end_date + timedelta(days=1))
 
     return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count()

app/dao/daily_sorted_letter_dao.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+from sqlalchemy.dialects.postgresql import insert
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import DailySortedLetter
+
+
+def dao_get_daily_sorted_letter_by_billing_day(billing_day):
+    return DailySortedLetter.query.filter_by(
+        billing_day=billing_day
+    ).first()
+
+
+@transactional
+def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
+    '''
+    This uses the Postgres upsert to avoid race conditions when two threads try and insert
+    at the same row. The excluded object refers to values that we tried to insert but were
+    rejected.
+    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
+    '''
+    table = DailySortedLetter.__table__
+    stmt = insert(table).values(
+        billing_day=new_daily_sorted_letter.billing_day,
+        file_name=new_daily_sorted_letter.file_name,
+        unsorted_count=new_daily_sorted_letter.unsorted_count,
+        sorted_count=new_daily_sorted_letter.sorted_count)
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[table.c.billing_day, table.c.file_name],
+        set_={
+            'unsorted_count': stmt.excluded.unsorted_count,
+            'sorted_count': stmt.excluded.sorted_count,
+            'updated_at': datetime.utcnow()
+        }
+    )
+    db.session.connection().execute(stmt)

app/dao/dao_utils.py

@@ -1,41 +1,25 @@
 import itertools
-from contextlib import contextmanager
 from functools import wraps
 
 from app import db
 from app.history_meta import create_history
 
 
-def autocommit(func):
+def transactional(func):
     @wraps(func)
     def commit_or_rollback(*args, **kwargs):
+        from flask import current_app
         try:
             res = func(*args, **kwargs)
-
-            if not db.session.registry().transaction.nested:
-                db.session.commit()
-
+            db.session.commit()
             return res
-        except Exception:
+        except Exception as e:
+            current_app.logger.error(e)
             db.session.rollback()
             raise
     return commit_or_rollback
 
 
-@contextmanager
-def transaction():
-    try:
-        db.session.begin_nested()
-        yield
-        db.session.commit()
-
-        if not db.session.registry().transaction.nested:
-            db.session.commit()
-    except Exception:
-        db.session.rollback()
-        raise
-
-
 class VersionOptions():
 
     def __init__(self, model_class, history_class=None, must_write_history=True):
@@ -91,9 +75,3 @@ def version_class(*version_options):
 
 def dao_rollback():
     db.session.rollback()
-
-
-@autocommit
-def dao_save_object(obj):
-    # add/update object in db
-    db.session.add(obj)

app/dao/date_util.py

@@ -1,20 +1,16 @@
-from datetime import date, datetime, time, timedelta
+from datetime import datetime, timedelta, date, time
 
+from notifications_utils.timezones import convert_bst_to_utc
 import pytz
-from notifications_utils.timezones import (
-    convert_local_timezone_to_utc,
-    convert_utc_to_local_timezone,
-    local_timezone,
-)
 
 
 def get_months_for_financial_year(year):
     return [
-        convert_local_timezone_to_utc(month) for month in (
+        convert_bst_to_utc(month) for month in (
             get_months_for_year(4, 13, year)
             + get_months_for_year(1, 4, year + 1)
         )
-        if convert_local_timezone_to_utc(month) < datetime.now()
+        if convert_bst_to_utc(month) < datetime.now()
     ]
 
 
@@ -26,15 +22,6 @@ def get_financial_year(year):
     return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
 
 
-def get_financial_year_dates(year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    return (
-        convert_utc_to_local_timezone(year_start_datetime).date(),
-        convert_utc_to_local_timezone(year_end_datetime).date()
-    )
-
-
 def get_current_financial_year():
     now = datetime.utcnow()
     current_month = int(now.strftime('%-m'))
@@ -50,8 +37,8 @@ def get_april_fools(year):
      :param year: the year to calculate the April 1, 00:00 BST for
      :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00
     """
-    return local_timezone.localize(
-        datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(tzinfo=None)
+    return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
+        tzinfo=None)
 
 
 def get_month_start_and_end_date_in_utc(month_year):
@@ -64,7 +51,7 @@ def get_month_start_and_end_date_in_utc(month_year):
     _, num_days = calendar.monthrange(month_year.year, month_year.month)
     first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
     last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
-    return convert_local_timezone_to_utc(first_day), convert_local_timezone_to_utc(last_day)
+    return convert_bst_to_utc(first_day), convert_bst_to_utc(last_day)
 
 
 def get_current_financial_year_start_year():

app/dao/email_branding_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import EmailBranding
 
 
@@ -15,12 +15,12 @@ def dao_get_email_branding_by_name(email_branding_name):
     return EmailBranding.query.filter_by(name=email_branding_name).first()
 
 
-@autocommit
+@transactional
 def dao_create_email_branding(email_branding):
     db.session.add(email_branding)
 
 
-@autocommit
+@transactional
 def dao_update_email_branding(email_branding, **kwargs):
     for key, value in kwargs.items():
         setattr(email_branding, key, value or None)

app/dao/fact_notification_status_dao.py

@@ -1,104 +1,122 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time
 
-from sqlalchemy import Date, case, func
+from flask import current_app
+from notifications_utils.timezones import convert_bst_to_utc
+from sqlalchemy import case, func, Date
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import extract, literal
+from sqlalchemy.sql.expression import literal, extract
 from sqlalchemy.types import DateTime, Integer
 
 from app import db
-from app.dao.dao_utils import autocommit
 from app.models import (
-    KEY_TYPE_NORMAL,
-    KEY_TYPE_TEAM,
+    FactNotificationStatus,
     KEY_TYPE_TEST,
+    Notification,
     NOTIFICATION_CANCELLED,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_FAILED,
-    NOTIFICATION_PENDING,
-    NOTIFICATION_PERMANENT_FAILURE,
     NOTIFICATION_SENDING,
     NOTIFICATION_SENT,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    FactNotificationStatus,
-    Notification,
-    NotificationAllTimeView,
+    NOTIFICATION_PERMANENT_FAILURE,
     Service,
     Template,
 )
+from app.dao.dao_utils import transactional
 from app.utils import (
-    get_local_midnight_in_utc,
-    get_local_month_from_utc_column,
+    get_london_midnight_in_utc,
     midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
 )
 
 
-@autocommit
-def update_fact_notification_status(process_day, notification_type, service_id):
-    start_date = get_local_midnight_in_utc(process_day)
-    end_date = get_local_midnight_in_utc(process_day + timedelta(days=1))
+def fetch_notification_status_for_day(process_day, notification_type):
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
 
-    # delete any existing rows in case some no longer exist e.g. if all messages are sent
-    FactNotificationStatus.query.filter(
-        FactNotificationStatus.local_date == process_day,
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.service_id == service_id,
-    ).delete()
+    current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
 
+    all_data_for_process_day = []
+    services = Service.query.all()
+    # for each service query notifications or notification_history for the day, depending on their data retention
+    for service in services:
+        table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
+
+        data_for_service_and_type = query_for_fact_status_data(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+            notification_type=notification_type,
+            service_id=service.id
+        )
+
+        all_data_for_process_day += data_for_service_and_type
+
+    return all_data_for_process_day
+
+
+def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
     query = db.session.query(
-        literal(process_day).label("process_day"),
-        NotificationAllTimeView.template_id,
-        literal(service_id).label("service_id"),
-        func.coalesce(NotificationAllTimeView.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
-        literal(notification_type).label("notification_type"),
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status,
+        table.template_id,
+        table.service_id,
+        func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
+        table.key_type,
+        table.status,
         func.count().label('notification_count')
     ).filter(
-        NotificationAllTimeView.created_at >= start_date,
-        NotificationAllTimeView.created_at < end_date,
-        NotificationAllTimeView.notification_type == notification_type,
-        NotificationAllTimeView.service_id == service_id,
-        NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
+        table.created_at >= start_date,
+        table.created_at < end_date,
+        table.notification_type == notification_type,
+        table.service_id == service_id,
+        table.key_type != KEY_TYPE_TEST
     ).group_by(
-        NotificationAllTimeView.template_id,
-        NotificationAllTimeView.template_id,
+        table.template_id,
+        table.service_id,
         'job_id',
-        NotificationAllTimeView.key_type,
-        NotificationAllTimeView.status
+        table.key_type,
+        table.status
     )
+    return query.all()
 
-    db.session.connection().execute(
-        insert(FactNotificationStatus.__table__).from_select(
-            [
-                FactNotificationStatus.local_date,
-                FactNotificationStatus.template_id,
-                FactNotificationStatus.service_id,
-                FactNotificationStatus.job_id,
-                FactNotificationStatus.notification_type,
-                FactNotificationStatus.key_type,
-                FactNotificationStatus.notification_status,
-                FactNotificationStatus.notification_count
-            ],
-            query
+
+@transactional
+def update_fact_notification_status(data, process_day, notification_type):
+    table = FactNotificationStatus.__table__
+    FactNotificationStatus.query.filter(
+        FactNotificationStatus.bst_date == process_day,
+        FactNotificationStatus.notification_type == notification_type
+    ).delete()
+
+    for row in data:
+        stmt = insert(table).values(
+            bst_date=process_day,
+            template_id=row.template_id,
+            service_id=row.service_id,
+            job_id=row.job_id,
+            notification_type=notification_type,
+            key_type=row.key_type,
+            notification_status=row.status,
+            notification_count=row.notification_count,
         )
-    )
+        db.session.connection().execute(stmt)
 
 
 def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
     return db.session.query(
-        func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
+        func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
         FactNotificationStatus.notification_type,
         FactNotificationStatus.notification_status,
         func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
         FactNotificationStatus.service_id == service_id,
-        FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.local_date < end_date,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date < end_date,
         FactNotificationStatus.key_type != KEY_TYPE_TEST
     ).group_by(
-        func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
+        func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
         FactNotificationStatus.notification_type,
         FactNotificationStatus.notification_status
     ).all()
@@ -112,8 +130,8 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
         Notification.status.label('notification_status'),
         func.count().label('count')
     ).filter(
-        Notification.created_at >= get_local_midnight_in_utc(bst_day),
-        Notification.created_at < get_local_midnight_in_utc(bst_day + timedelta(days=1)),
+        Notification.created_at >= get_london_midnight_in_utc(bst_day),
+        Notification.created_at < get_london_midnight_in_utc(bst_day + timedelta(days=1)),
         Notification.service_id == service_id,
         Notification.key_type != KEY_TYPE_TEST
     ).group_by(
@@ -132,7 +150,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
         FactNotificationStatus.notification_count.label('count')
     ).filter(
         FactNotificationStatus.service_id == service_id,
-        FactNotificationStatus.local_date >= start_date,
+        FactNotificationStatus.bst_date >= start_date,
         FactNotificationStatus.key_type != KEY_TYPE_TEST
     )
 
@@ -142,7 +160,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
         *([Notification.template_id] if by_template else []),
         func.count().label('count')
     ).filter(
-        Notification.created_at >= get_local_midnight_in_utc(now),
+        Notification.created_at >= get_london_midnight_in_utc(now),
         Notification.service_id == service_id,
         Notification.key_type != KEY_TYPE_TEST
     ).group_by(
@@ -156,7 +174,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
     query = db.session.query(
         *([
             Template.name.label("template_name"),
-            False,  # TODO: this is related to is_precompiled_letter
+            Template.is_precompiled_letter,
             all_stats_table.c.template_id
         ] if by_template else []),
         all_stats_table.c.notification_type,
@@ -168,7 +186,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
         query = query.filter(all_stats_table.c.template_id == Template.id)
 
     return query.group_by(
-        *([Template.name, all_stats_table.c.template_id] if by_template else []),
+        *([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []),
         all_stats_table.c.notification_type,
         all_stats_table.c.status,
     ).all()
@@ -181,14 +199,14 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
         FactNotificationStatus.key_type.label('key_type'),
         func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
-        FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.local_date <= end_date
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date
     ).group_by(
         FactNotificationStatus.notification_type,
         FactNotificationStatus.notification_status,
         FactNotificationStatus.key_type,
     )
-    today = get_local_midnight_in_utc(datetime.utcnow())
+    today = get_london_midnight_in_utc(datetime.utcnow())
     if start_date <= datetime.utcnow().date() <= end_date:
         stats_for_today = db.session.query(
             Notification.notification_type.cast(db.Text).label('notification_type'),
@@ -245,8 +263,8 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
         FactNotificationStatus.notification_status.label('status'),
         func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
-        FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.local_date <= end_date,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date,
         FactNotificationStatus.service_id == Service.id,
     ).group_by(
         FactNotificationStatus.service_id.label('service_id'),
@@ -265,7 +283,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
         stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
 
     if start_date <= datetime.utcnow().date() <= end_date:
-        today = get_local_midnight_in_utc(datetime.utcnow())
+        today = get_london_midnight_in_utc(datetime.utcnow())
         subquery = db.session.query(
             Notification.notification_type.cast(db.Text).label('notification_type'),
             Notification.status.label('status'),
@@ -333,37 +351,40 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
         FactNotificationStatus.template_id.label('template_id'),
         Template.name.label('name'),
         Template.template_type.label('template_type'),
-        extract('month', FactNotificationStatus.local_date).label('month'),
-        extract('year', FactNotificationStatus.local_date).label('year'),
+        Template.is_precompiled_letter.label('is_precompiled_letter'),
+        extract('month', FactNotificationStatus.bst_date).label('month'),
+        extract('year', FactNotificationStatus.bst_date).label('year'),
         func.sum(FactNotificationStatus.notification_count).label('count')
     ).join(
         Template, FactNotificationStatus.template_id == Template.id
     ).filter(
         FactNotificationStatus.service_id == service_id,
-        FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.local_date <= end_date,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date,
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
         FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
     ).group_by(
         FactNotificationStatus.template_id,
         Template.name,
         Template.template_type,
-        extract('month', FactNotificationStatus.local_date).label('month'),
-        extract('year', FactNotificationStatus.local_date).label('year'),
+        Template.is_precompiled_letter,
+        extract('month', FactNotificationStatus.bst_date).label('month'),
+        extract('year', FactNotificationStatus.bst_date).label('year'),
     ).order_by(
-        extract('year', FactNotificationStatus.local_date),
-        extract('month', FactNotificationStatus.local_date),
+        extract('year', FactNotificationStatus.bst_date),
+        extract('month', FactNotificationStatus.bst_date),
         Template.name
     )
 
     if start_date <= datetime.utcnow() <= end_date:
-        today = get_local_midnight_in_utc(datetime.utcnow())
-        month = get_local_month_from_utc_column(Notification.created_at)
+        today = get_london_midnight_in_utc(datetime.utcnow())
+        month = get_london_month_from_utc_column(Notification.created_at)
 
         stats_for_today = db.session.query(
             Notification.template_id.label('template_id'),
             Template.name.label('name'),
             Template.template_type.label('template_type'),
+            Template.is_precompiled_letter.label('is_precompiled_letter'),
             extract('month', month).label('month'),
             extract('year', month).label('year'),
             func.count().label('count')
@@ -386,6 +407,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
         query = db.session.query(
             all_stats_table.c.template_id,
             all_stats_table.c.name,
+            all_stats_table.c.is_precompiled_letter,
             all_stats_table.c.template_type,
             func.cast(all_stats_table.c.month, Integer).label('month'),
             func.cast(all_stats_table.c.year, Integer).label('year'),
@@ -393,6 +415,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
         ).group_by(
             all_stats_table.c.template_id,
             all_stats_table.c.name,
+            all_stats_table.c.is_precompiled_letter,
             all_stats_table.c.template_type,
             all_stats_table.c.month,
             all_stats_table.c.year,
@@ -406,43 +429,27 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
     return query.all()
 
 
-def get_total_notifications_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactNotificationStatus.local_date.cast(db.Text).label("local_date"),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('emails'),
-        func.sum(case(
-            [
-                (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
-            ],
-            else_=0)).label('sms'),
+def get_total_sent_notifications_for_day_and_type(day, notification_type):
+    result = db.session.query(
+        func.sum(FactNotificationStatus.notification_count).label('count')
     ).filter(
+        FactNotificationStatus.notification_type == notification_type,
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
-    ).group_by(
-        FactNotificationStatus.local_date
-    ).order_by(
-        FactNotificationStatus.local_date
-    )
-    if start_date and end_date:
-        query = query.filter(
-            FactNotificationStatus.local_date >= start_date,
-            FactNotificationStatus.local_date <= end_date
-        )
-    return query.all()
+        FactNotificationStatus.bst_date == day,
+    ).scalar()
+
+    return result or 0
 
 
 def fetch_monthly_notification_statuses_per_service(start_date, end_date):
     return db.session.query(
-        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date).label('date_created'),
+        func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'),
         Service.id.label('service_id'),
         Service.name.label('service_name'),
         FactNotificationStatus.notification_type,
         func.sum(case(
             [
-                (FactNotificationStatus.notification_status.in_([NOTIFICATION_SENDING, NOTIFICATION_PENDING]),
+                (FactNotificationStatus.notification_status == NOTIFICATION_SENDING,
                  FactNotificationStatus.notification_count)
             ],
             else_=0)).label('count_sending'),
@@ -484,15 +491,15 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
         FactNotificationStatus.key_type != KEY_TYPE_TEST,
         Service.research_mode.is_(False),
         Service.restricted.is_(False),
-        FactNotificationStatus.local_date >= start_date,
-        FactNotificationStatus.local_date <= end_date,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date,
     ).group_by(
         Service.id,
         Service.name,
-        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
+        func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
         FactNotificationStatus.notification_type,
     ).order_by(
-        func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
+        func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
         Service.id,
         FactNotificationStatus.notification_type,
     ).all()

app/dao/fact_processing_time_dao.py

@@ -1,53 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy.sql.expression import case
-
-from app import db
-from app.dao.dao_utils import autocommit
-from app.models import FactProcessingTime
-
-
-@autocommit
-def insert_update_processing_time(processing_time):
-    '''
-    This uses the Postgres upsert to avoid race conditions when two threads try and insert
-    at the same row. The excluded object refers to values that we tried to insert but were
-    rejected.
-    http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
-    '''
-    table = FactProcessingTime.__table__
-    stmt = insert(table).values(
-        local_date=processing_time.local_date,
-        messages_total=processing_time.messages_total,
-        messages_within_10_secs=processing_time.messages_within_10_secs
-    )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.local_date],
-        set_={
-            'messages_total': stmt.excluded.messages_total,
-            'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
-            'updated_at': datetime.utcnow()
-        }
-    )
-    db.session.connection().execute(stmt)
-
-
-def get_processing_time_percentage_for_date_range(start_date, end_date):
-    query = db.session.query(
-        FactProcessingTime.local_date.cast(db.Text).label("date"),
-        FactProcessingTime.messages_total,
-        FactProcessingTime.messages_within_10_secs,
-        case([
-            (
-                FactProcessingTime.messages_total > 0,
-                ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast(db.Float)) * 100)
-            ),
-            (FactProcessingTime.messages_total == 0, 100.0)
-        ]).label("percentage")
-    ).filter(
-        FactProcessingTime.local_date >= start_date,
-        FactProcessingTime.local_date <= end_date
-    ).order_by(FactProcessingTime.local_date)
-
-    return query.all()

app/dao/inbound_numbers_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import InboundNumber
 
 
@@ -19,13 +19,13 @@ def dao_get_inbound_number(inbound_number_id):
     return InboundNumber.query.filter(InboundNumber.id == inbound_number_id).first()
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_to_service(service_id, inbound_number):
     inbound_number.service_id = service_id
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_set_inbound_number_active_flag(service_id, active):
     inbound_number = InboundNumber.query.filter(InboundNumber.service_id == service_id).first()
     inbound_number.active = active
@@ -33,7 +33,7 @@ def dao_set_inbound_number_active_flag(service_id, active):
     db.session.add(inbound_number)
 
 
-@autocommit
+@transactional
 def dao_allocate_number_for_service(service_id, inbound_number_id):
     updated = InboundNumber.query.filter_by(
         id=inbound_number_id,

app/dao/inbound_sms_dao.py

@@ -1,21 +1,16 @@
 from flask import current_app
-from sqlalchemy import and_, desc
-from sqlalchemy.dialects.postgresql import insert
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import desc, and_
 from sqlalchemy.orm import aliased
+from sqlalchemy.dialects.postgresql import insert
 
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    InboundSms,
-    InboundSmsHistory,
-    Service,
-    ServiceDataRetention,
-)
+from app.dao.dao_utils import transactional
+from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention, SMS_TYPE
 from app.utils import midnight_n_days_ago
 
 
-@autocommit
+@transactional
 def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
@@ -71,13 +66,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days):
 def _insert_inbound_sms_history(subquery, query_limit=10000):
     offset = 0
     inbound_sms_query = db.session.query(
-        InboundSms.id,
-        InboundSms.created_at,
-        InboundSms.service_id,
-        InboundSms.notify_number,
-        InboundSms.provider_date,
-        InboundSms.provider_reference,
-        InboundSms.provider
+        *[x.name for x in InboundSmsHistory.__table__.c]
     ).filter(InboundSms.id.in_(subquery))
     inbound_sms_count = inbound_sms_query.count()
 
@@ -119,7 +108,8 @@ def _delete_inbound_sms(datetime_to_delete_from, query_filter):
     return deleted
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def delete_inbound_sms_older_than_retention():
     current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 

app/dao/invited_org_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedOrganisationUser
 
 

app/dao/invited_user_dao.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
-
 from app import db
+
 from app.models import InvitedUser
 
 
@@ -9,7 +9,7 @@ def save_invited_user(invited_user):
     db.session.commit()
 
 
-def get_invited_user_by_service_and_id(service_id, invited_user_id):
+def get_invited_user(service_id, invited_user_id):
     return InvitedUser.query.filter_by(service_id=service_id, id=invited_user_id).one()
 
 

app/dao/jobs_dao.py

@@ -2,22 +2,37 @@ import uuid
 from datetime import datetime, timedelta
 
 from flask import current_app
-from sqlalchemy import and_, asc, desc, func
+from notifications_utils.letter_timings import letter_can_be_cancelled, CANCELLABLE_JOB_LETTER_STATUSES
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy import (
+    asc,
+    desc,
+    func,
+    and_
+)
 
 from app import db
+from app.dao.dao_utils import transactional
+from app.dao.templates_dao import dao_get_template_by_id
+from app.utils import midnight_n_days_ago
+
 from app.models import (
+    Job,
     JOB_STATUS_FINISHED,
     JOB_STATUS_PENDING,
     JOB_STATUS_SCHEDULED,
-    FactNotificationStatus,
-    Job,
+    LETTER_TYPE,
     Notification,
-    ServiceDataRetention,
     Template,
+    ServiceDataRetention,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_CANCELLED,
+    JOB_STATUS_CANCELLED,
+    FactNotificationStatus
 )
-from app.utils import midnight_n_days_ago
 
 
+@statsd(namespace="dao")
 def dao_get_notification_outcomes_for_job(service_id, job_id):
     notification_statuses = db.session.query(
         func.count(Notification.status).label('count'), Notification.status
@@ -43,15 +58,7 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
     return Job.query.filter_by(service_id=service_id, id=job_id).one()
 
 
-def dao_get_jobs_by_service_id(
-    service_id,
-    *,
-    limit_days=None,
-    page=1,
-    page_size=50,
-    statuses=None,
-    contact_list_id=None,
-):
+def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
     query_filter = [
         Job.service_id == service_id,
         Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
@@ -63,26 +70,12 @@ def dao_get_jobs_by_service_id(
         query_filter.append(
             Job.job_status.in_(statuses)
         )
-    if contact_list_id is not None:
-        query_filter.append(Job.contact_list_id == contact_list_id)
     return Job.query \
         .filter(*query_filter) \
         .order_by(Job.processing_started.desc(), Job.created_at.desc()) \
         .paginate(page=page, per_page=page_size)
 
 
-def dao_get_scheduled_job_stats(
-    service_id,
-):
-    return db.session.query(
-        func.count(Job.id),
-        func.min(Job.scheduled_for),
-    ).filter(
-        Job.service_id == service_id,
-        Job.job_status == JOB_STATUS_SCHEDULED,
-    ).one()
-
-
 def dao_get_job_by_id(job_id):
     return Job.query.filter_by(id=job_id).one()
 
@@ -173,12 +166,47 @@ def dao_get_jobs_older_than_data_retention(notification_types):
     return jobs
 
 
+@transactional
+def dao_cancel_letter_job(job):
+    number_of_notifications_cancelled = Notification.query.filter(
+        Notification.job_id == job.id
+    ).update({'status': NOTIFICATION_CANCELLED,
+              'updated_at': datetime.utcnow(),
+              'billable_units': 0})
+    job.job_status = JOB_STATUS_CANCELLED
+    dao_update_job(job)
+    return number_of_notifications_cancelled
+
+
+def can_letter_job_be_cancelled(job):
+    template = dao_get_template_by_id(job.template_id)
+    if template.template_type != LETTER_TYPE:
+        return False, "Only letter jobs can be cancelled through this endpoint. This is not a letter job."
+
+    notifications = Notification.query.filter(
+        Notification.job_id == job.id
+    ).all()
+    count_notifications = len(notifications)
+    if job.job_status != JOB_STATUS_FINISHED or count_notifications != job.notification_count:
+        return False, "We are still processing these letters, please try again in a minute."
+    count_cancellable_notifications = len([
+        n for n in notifications if n.status in CANCELLABLE_JOB_LETTER_STATUSES
+    ])
+    if count_cancellable_notifications != job.notification_count or not letter_can_be_cancelled(
+        NOTIFICATION_CREATED, job.created_at
+    ):
+        return False, "It’s too late to cancel sending, these letters have already been sent."
+
+    return True, None
+
+
 def find_jobs_with_missing_rows():
-    # Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
-    # Using 20 minutes as a condition seems reasonable.
-    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=20)
+    # Jobs can be a maximum of 50,000 rows. It typically takes 5 minutes to create all those notifications.
+    # Using 10 minutes as a condition seems reasonable.
+    ten_minutes_ago = datetime.utcnow() - timedelta(minutes=10)
     yesterday = datetime.utcnow() - timedelta(days=1)
     jobs_with_rows_missing = db.session.query(
+        func.count(Notification.id).label('actual_count'),
         Job
     ).filter(
         Job.job_status == JOB_STATUS_FINISHED,

app/dao/letter_branding_dao.py

@@ -0,0 +1,29 @@
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import LetterBranding
+
+
+def dao_get_letter_branding_by_id(letter_branding_id):
+    return LetterBranding.query.filter(LetterBranding.id == letter_branding_id).one()
+
+
+def dao_get_letter_branding_by_name(letter_branding_name):
+    return LetterBranding.query.filter_by(name=letter_branding_name).first()
+
+
+def dao_get_all_letter_branding():
+    return LetterBranding.query.order_by(LetterBranding.name).all()
+
+
+@transactional
+def dao_create_letter_branding(letter_branding):
+    db.session.add(letter_branding)
+
+
+@transactional
+def dao_update_letter_branding(letter_branding_id, **kwargs):
+    letter_branding = LetterBranding.query.get(letter_branding_id)
+    for key, value in kwargs.items():
+        setattr(letter_branding, key, value or None)
+    db.session.add(letter_branding)
+    return letter_branding

app/dao/notifications_dao.py

@@ -1,45 +1,77 @@
-from datetime import datetime, timedelta
+import functools
+from itertools import groupby
+from operator import attrgetter
+from datetime import (
+    datetime,
+    timedelta,
+)
 
+from boto.exception import BotoClientError
 from flask import current_app
-from notifications_utils.international_billing_rates import (
-    INTERNATIONAL_BILLING_RATES,
-)
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
     validate_and_format_email_address,
+    InvalidEmailError,
+    try_validate_and_format_phone_number
 )
-from sqlalchemy import asc, desc, func, or_, union
+from notifications_utils.statsd_decorators import statsd
+from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
+from sqlalchemy import (desc, func, asc, and_, or_)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
+from sqlalchemy.dialects.postgresql import insert
 from werkzeug.datastructures import MultiDict
 
-from app import create_uuid, db
-from app.dao.dao_utils import autocommit
+from app import db, create_uuid
+from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
+from app.dao.dao_utils import transactional
+from app.errors import InvalidRequest
+from app.letters.utils import get_letter_pdf_filename
 from app.models import (
-    EMAIL_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_CREATED,
-    NOTIFICATION_PENDING,
-    NOTIFICATION_PENDING_VIRUS_CHECK,
-    NOTIFICATION_PERMANENT_FAILURE,
-    NOTIFICATION_SENDING,
-    NOTIFICATION_SENT,
-    NOTIFICATION_TEMPORARY_FAILURE,
-    SMS_TYPE,
     FactNotificationStatus,
     Notification,
     NotificationHistory,
+    ProviderDetails,
+    ScheduledNotification,
+    KEY_TYPE_NORMAL,
+    KEY_TYPE_TEST,
+    LETTER_TYPE,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_SENDING,
+    NOTIFICATION_PENDING,
+    NOTIFICATION_PENDING_VIRUS_CHECK,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+    NOTIFICATION_SENT,
+    SMS_TYPE,
+    EMAIL_TYPE,
+    ServiceDataRetention,
+    Service,
 )
-from app.utils import (
-    escape_special_characters,
-    get_local_midnight_in_utc,
-    midnight_n_days_ago,
-)
+from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters
 
 
+@statsd(namespace="dao")
+def dao_get_last_template_usage(template_id, template_type, service_id):
+    # By adding the service_id to the filter the performance of the query is greatly improved.
+    # Using a max(Notification.created_at) is better than order by and limit one.
+    # But the effort to change the endpoint to return a datetime only is more than the gain.
+    return Notification.query.filter(
+        Notification.template_id == template_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type == template_type,
+        Notification.service_id == service_id
+    ).order_by(
+        desc(Notification.created_at)
+    ).first()
+
+
+@statsd(namespace="dao")
 def dao_get_last_date_template_was_used(template_id, service_id):
     last_date_from_notifications = db.session.query(
         functions.max(Notification.created_at)
@@ -53,7 +85,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
         return last_date_from_notifications
 
     last_date = db.session.query(
-        functions.max(FactNotificationStatus.local_date)
+        functions.max(FactNotificationStatus.bst_date)
     ).filter(
         FactNotificationStatus.template_id == template_id,
         FactNotificationStatus.key_type != KEY_TYPE_TEST
@@ -62,7 +94,8 @@ def dao_get_last_date_template_was_used(template_id, service_id):
     return last_date
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_create_notification(notification):
     if not notification.id:
         # need to populate defaulted fields before we create the notification history object
@@ -73,28 +106,28 @@ def dao_create_notification(notification):
     db.session.add(notification)
 
 
-def country_records_delivery(phone_prefix):
-    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
-    return dlr and dlr.lower() == 'yes'
-
-
 def _decide_permanent_temporary_failure(current_status, status):
+    # Firetext will send pending, then send either succes or fail.
     # If we go from pending to delivered we need to set failure type as temporary-failure
     if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
         status = NOTIFICATION_TEMPORARY_FAILURE
     return status
 
 
-def _update_notification_status(notification, status, provider_response=None):
+def country_records_delivery(phone_prefix):
+    dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
+    return dlr and dlr.lower() == 'yes'
+
+
+def _update_notification_status(notification, status):
     status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
     notification.status = status
-    if provider_response:
-        notification.provider_response = provider_response
     dao_update_notification(notification)
     return notification
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def update_notification_status_by_id(notification_id, status, sent_by=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
 
@@ -115,11 +148,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None):
         _duplicate_update_warning(notification, status)
         return None
 
-    if (
-        notification.notification_type == SMS_TYPE
-        and notification.international
-        and not country_records_delivery(notification.phone_prefix)
-    ):
+    if notification.international and not country_records_delivery(notification.phone_prefix):
         return None
     if not notification.sent_by and sent_by:
         notification.sent_by = sent_by
@@ -129,9 +158,10 @@ def update_notification_status_by_id(notification_id, status, sent_by=None):
     )
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def update_notification_status_by_reference(reference, status):
-    # this is used to update emails
+    # this is used to update letters and emails
     notification = Notification.query.filter(Notification.reference == reference).first()
 
     if not notification:
@@ -151,12 +181,19 @@ def update_notification_status_by_reference(reference, status):
     )
 
 
-@autocommit
+@statsd(namespace="dao")
+@transactional
 def dao_update_notification(notification):
     notification.updated_at = datetime.utcnow()
     db.session.add(notification)
 
 
+@statsd(namespace="dao")
+def get_notification_for_job(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
+@statsd(namespace="dao")
 def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page_size=None):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -168,10 +205,12 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
     )
 
 
+@statsd(namespace="dao")
 def dao_get_notification_count_for_job_id(*, job_id):
     return Notification.query.filter_by(job_id=job_id).count()
 
 
+@statsd(namespace="dao")
 def get_notification_with_personalisation(service_id, notification_id, key_type):
     filter_dict = {'service_id': service_id, 'id': notification_id}
     if key_type:
@@ -180,6 +219,7 @@ def get_notification_with_personalisation(service_id, notification_id, key_type)
     return Notification.query.filter_by(**filter_dict).options(joinedload('template')).one()
 
 
+@statsd(namespace="dao")
 def get_notification_by_id(notification_id, service_id=None, _raise=False):
     filters = [Notification.id == notification_id]
 
@@ -191,6 +231,11 @@ def get_notification_by_id(notification_id, service_id=None, _raise=False):
     return query.one() if _raise else query.first()
 
 
+def get_notifications(filter_dict=None):
+    return _filter_query(Notification.query, filter_dict=filter_dict)
+
+
+@statsd(namespace="dao")
 def get_notifications_for_service(
         service_id,
         filter_dict=None,
@@ -204,8 +249,7 @@ def get_notifications_for_service(
         include_from_test_key=False,
         older_than=None,
         client_reference=None,
-        include_one_off=True,
-        error_out=True
+        include_one_off=True
 ):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -244,8 +288,7 @@ def get_notifications_for_service(
     return query.order_by(desc(Notification.created_at)).paginate(
         page=page,
         per_page=page_size,
-        count=count_pages,
-        error_out=error_out,
+        count=count_pages
     )
 
 
@@ -269,132 +312,265 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
-@autocommit
-def insert_notification_history_delete_notifications(
-    notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000
-):
-    """
-    Delete up to 50,000 notifications that are past retention for a notification type and service.
+@statsd(namespace="dao")
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=10000):
+    current_app.logger.info(
+        'Deleting {} notifications for services with flexible data retention'.format(notification_type))
 
-
-    Steps are as follows:
-
-    Create a temporary notifications table
-    Populate that table with up to 50k notifications that are to be deleted. (Note: no specified order)
-    Insert everything in the temp table into notification history
-    Delete from notifications if notification id is in the temp table
-    Drop the temp table (automatically when the transaction commits)
-
-    Temporary tables are in a separate postgres schema, and only visible to the current session (db connection,
-    in a celery task there's one connection per thread.)
-    """
-    # Setting default query limit to 50,000 which take about 48 seconds on current table size
-    # 10, 000 took 11s and 100,000 took 1 min 30 seconds.
-    select_into_temp_table = """
-         CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
-         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
-             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
-             client_reference, international, phone_prefix, rate_multiplier, notification_status,
-              created_by_id, document_download_count
-          FROM notifications
-        WHERE service_id = :service_id
-          AND notification_type = :notification_type
-          AND created_at < :timestamp_to_delete_backwards_from
-          AND key_type in ('normal', 'team')
-        limit :qry_limit
-        """
-    # Insert into NotificationHistory if the row already exists do nothing.
-    insert_query = """
-        insert into notification_history
-         SELECT * from NOTIFICATION_ARCHIVE
-          ON CONFLICT ON CONSTRAINT notification_history_pkey
-          DO NOTHING
-    """
-    delete_query = """
-        DELETE FROM notifications
-        where id in (select id from NOTIFICATION_ARCHIVE)
-    """
-    input_params = {
-        "service_id": service_id,
-        "notification_type": notification_type,
-        "timestamp_to_delete_backwards_from": timestamp_to_delete_backwards_from,
-        "qry_limit": qry_limit
-    }
-
-    db.session.execute(select_into_temp_table, input_params)
-
-    result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
-
-    db.session.execute(insert_query)
-
-    db.session.execute(delete_query)
-
-    return result
-
-
-def move_notifications_to_notification_history(
-    notification_type,
-    service_id,
-    timestamp_to_delete_backwards_from,
-    qry_limit=50000
-):
+    flexible_data_retention = ServiceDataRetention.query.filter(
+        ServiceDataRetention.notification_type == notification_type
+    ).all()
     deleted = 0
-    delete_count_per_call = 1
-    while delete_count_per_call > 0:
-        delete_count_per_call = insert_notification_history_delete_notifications(
-            notification_type=notification_type,
-            service_id=service_id,
-            timestamp_to_delete_backwards_from=timestamp_to_delete_backwards_from,
-            qry_limit=qry_limit
-        )
-        deleted += delete_count_per_call
+    for f in flexible_data_retention:
+        days_of_retention = get_london_midnight_in_utc(
+            convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)
 
-    # Deleting test Notifications, test notifications are not persisted to NotificationHistory
-    Notification.query.filter(
-        Notification.notification_type == notification_type,
-        Notification.service_id == service_id,
-        Notification.created_at < timestamp_to_delete_backwards_from,
-        Notification.key_type == KEY_TYPE_TEST
-    ).delete(synchronize_session=False)
-    db.session.commit()
+        if notification_type == LETTER_TYPE:
+            _delete_letters_from_s3(
+                notification_type, f.service_id, days_of_retention, qry_limit
+            )
+
+        insert_update_notification_history(notification_type, days_of_retention, f.service_id)
+
+        current_app.logger.info(
+            "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
+        deleted += _delete_notifications(notification_type, days_of_retention, f.service_id, qry_limit)
+
+    current_app.logger.info(
+        'Deleting {} notifications for services without flexible data retention'.format(notification_type))
+
+    seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
+    services_with_data_retention = [x.service_id for x in flexible_data_retention]
+    service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
+
+    for service_id in service_ids_to_purge:
+        if notification_type == LETTER_TYPE:
+            _delete_letters_from_s3(
+                notification_type, service_id, seven_days_ago, qry_limit
+            )
+        insert_update_notification_history(notification_type, seven_days_ago, service_id)
+        deleted += _delete_notifications(notification_type, seven_days_ago, service_id, qry_limit)
+
+    current_app.logger.info('Finished deleting {} notifications'.format(notification_type))
 
     return deleted
 
 
-@autocommit
+def _delete_notifications(notification_type, date_to_delete_from, service_id, query_limit):
+    subquery = db.session.query(
+        Notification.id
+    ).join(NotificationHistory, NotificationHistory.id == Notification.id).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+    ).limit(query_limit).subquery()
+
+    deleted = _delete_for_query(subquery)
+
+    subquery_for_test_keys = db.session.query(
+        Notification.id
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+        Notification.key_type == KEY_TYPE_TEST
+    ).limit(query_limit).subquery()
+
+    deleted += _delete_for_query(subquery_for_test_keys)
+
+    return deleted
+
+
+def _delete_for_query(subquery):
+    number_deleted = db.session.query(Notification).filter(
+        Notification.id.in_(subquery)).delete(synchronize_session='fetch')
+    deleted = number_deleted
+    db.session.commit()
+    while number_deleted > 0:
+        number_deleted = db.session.query(Notification).filter(
+            Notification.id.in_(subquery)).delete(synchronize_session='fetch')
+        deleted += number_deleted
+        db.session.commit()
+    return deleted
+
+
+def insert_update_notification_history(notification_type, date_to_delete_from, service_id, query_limit=10000):
+    offset = 0
+
+    notification_query = db.session.query(
+        *[x.name for x in NotificationHistory.__table__.c]
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+        Notification.key_type != KEY_TYPE_TEST
+    )
+    notifications_count = notification_query.count()
+
+    while offset < notifications_count:
+        stmt = insert(NotificationHistory).from_select(
+            NotificationHistory.__table__.c,
+            notification_query.limit(query_limit).offset(offset)
+        )
+
+        stmt = stmt.on_conflict_do_update(
+            constraint="notification_history_pkey",
+            set_={
+                "notification_status": stmt.excluded.status,
+                "reference": stmt.excluded.reference,
+                "billable_units": stmt.excluded.billable_units,
+                "updated_at": stmt.excluded.updated_at,
+                "sent_at": stmt.excluded.sent_at,
+                "sent_by": stmt.excluded.sent_by
+            }
+        )
+        db.session.connection().execute(stmt)
+        db.session.commit()
+
+        offset += query_limit
+
+
+def _delete_letters_from_s3(
+        notification_type, service_id, date_to_delete_from, query_limit
+):
+    letters_to_delete_from_s3 = db.session.query(
+        Notification
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.created_at < date_to_delete_from,
+        Notification.service_id == service_id
+    ).limit(query_limit).all()
+    for letter in letters_to_delete_from_s3:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+        # I don't think we need this anymore, we should update the query to get letters sent 7 days ago
+        if letter.sent_at:
+            prefix = get_letter_pdf_filename(reference=letter.reference,
+                                             crown=letter.service.crown,
+                                             sending_date=letter.created_at,
+                                             dont_use_sending_date=letter.key_type == KEY_TYPE_TEST,
+                                             postage=letter.postage)
+            s3_objects = get_s3_bucket_objects(bucket_name=bucket_name, subfolder=prefix)
+            for s3_object in s3_objects:
+                try:
+                    remove_s3_object(bucket_name, s3_object['Key'])
+                except BotoClientError:
+                    current_app.logger.exception(
+                        "Could not delete S3 object with filename: {}".format(s3_object['Key']))
+
+
+@statsd(namespace="dao")
+@transactional
 def dao_delete_notifications_by_id(notification_id):
     db.session.query(Notification).filter(
         Notification.id == notification_id
     ).delete(synchronize_session='fetch')
 
 
-def dao_timeout_notifications(cutoff_time, limit=100000):
-    """
-    Set email and SMS notifications (only) to "temporary-failure" status
-    if they're still sending from before the specified cutoff_time.
-    """
-    updated_at = datetime.utcnow()
-    current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
-    new_status = NOTIFICATION_TEMPORARY_FAILURE
-
+def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
     notifications = Notification.query.filter(
-        Notification.created_at < cutoff_time,
+        Notification.created_at < timeout_start,
         Notification.status.in_(current_statuses),
-        Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE])
-    ).limit(limit).all()
-
+        Notification.notification_type != LETTER_TYPE
+    ).all()
     Notification.query.filter(
-        Notification.id.in_([n.id for n in notifications]),
+        Notification.created_at < timeout_start,
+        Notification.status.in_(current_statuses),
+        Notification.notification_type != LETTER_TYPE
     ).update(
         {'status': new_status, 'updated_at': updated_at},
         synchronize_session=False
     )
-
-    db.session.commit()
     return notifications
 
 
-@autocommit
+def dao_timeout_notifications(timeout_period_in_seconds):
+    """
+    Timeout SMS and email notifications by the following rules:
+
+    we never sent the notification to the provider for some reason
+        created -> technical-failure
+
+    the notification was sent to the provider but there was not a delivery receipt
+        sending -> temporary-failure
+        pending -> temporary-failure
+
+    Letter notifications are not timed out
+    """
+    timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
+    updated_at = datetime.utcnow()
+    timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
+
+    # Notifications still in created status are marked with a technical-failure:
+    technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
+
+    # Notifications still in sending or pending status are marked with a temporary-failure:
+    temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
+                                              NOTIFICATION_TEMPORARY_FAILURE)
+
+    db.session.commit()
+
+    return technical_failure_notifications, temporary_failure_notifications
+
+
+def is_delivery_slow_for_providers(
+        created_at,
+        threshold,
+        delivery_time,
+):
+    """
+    Returns a dict of providers and whether they are currently slow or not. eg:
+    {
+        'mmg': True,
+        'firetext': False
+    }
+    """
+    slow_notification_counts = db.session.query(
+        ProviderDetails.identifier,
+        case(
+            [(
+                Notification.status == NOTIFICATION_DELIVERED,
+                (Notification.updated_at - Notification.sent_at) >= delivery_time
+            )],
+            else_=(datetime.utcnow() - Notification.sent_at) >= delivery_time
+        ).label("slow"),
+        func.count().label('count')
+    ).select_from(
+        ProviderDetails
+    ).outerjoin(
+        Notification, and_(
+            Notification.sent_by == ProviderDetails.identifier,
+            Notification.created_at >= created_at,
+            Notification.sent_at.isnot(None),
+            Notification.status.in_([NOTIFICATION_DELIVERED, NOTIFICATION_PENDING, NOTIFICATION_SENDING]),
+            Notification.key_type != KEY_TYPE_TEST
+        )
+    ).filter(
+        ProviderDetails.notification_type == 'sms',
+        ProviderDetails.active
+    ).order_by(
+        ProviderDetails.identifier
+    ).group_by(
+        ProviderDetails.identifier,
+        "slow"
+    )
+
+    slow_providers = {}
+    for provider, rows in groupby(slow_notification_counts, key=attrgetter('identifier')):
+        rows = list(rows)
+        total_notifications = sum(row.count for row in rows)
+        slow_notifications = sum(row.count for row in rows if row.slow)
+
+        slow_providers[provider] = (slow_notifications / total_notifications >= threshold)
+
+        current_app.logger.info("Slow delivery notifications count for provider {}: {} out of {}. Ratio {}".format(
+            provider, slow_notifications, total_notifications, slow_notifications / total_notifications
+        ))
+
+    return slow_providers
+
+
+@statsd(namespace="dao")
+@transactional
 def dao_update_notifications_by_reference(references, update_dict):
     updated_count = Notification.query.filter(
         Notification.reference.in_(references)
@@ -415,15 +591,8 @@ def dao_update_notifications_by_reference(references, update_dict):
     return updated_count, updated_history_count
 
 
-def dao_get_notifications_by_recipient_or_reference(
-    service_id,
-    search_term,
-    notification_type=None,
-    statuses=None,
-    page=1,
-    page_size=None,
-    error_out=True,
-):
+@statsd(namespace="dao")
+def dao_get_notifications_by_recipient_or_reference(service_id, search_term, notification_type=None, statuses=None):
 
     if notification_type == SMS_TYPE:
         normalised = try_validate_and_format_phone_number(search_term)
@@ -439,17 +608,11 @@ def dao_get_notifications_by_recipient_or_reference(
         except InvalidEmailError:
             normalised = search_term.lower()
 
-    elif notification_type is None:
-        # This happens when a notification type isn’t provided (this will
-        # happen if a user doesn’t have permission to see the dashboard)
-        # because email addresses and phone numbers will never be stored
-        # with spaces either.
-        normalised = ''.join(search_term.split()).lower()
+    elif notification_type == LETTER_TYPE:
+        raise InvalidRequest("Only email and SMS can use search by recipient", 400)
 
     else:
-        raise TypeError(
-            f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, or None'
-        )
+        normalised = search_term.lower()
 
     normalised = escape_special_characters(normalised)
     search_term = escape_special_characters(search_term)
@@ -468,42 +631,76 @@ def dao_get_notifications_by_recipient_or_reference(
     if notification_type:
         filters.append(Notification.notification_type == notification_type)
 
-    results = db.session.query(Notification)\
-        .filter(*filters)\
-        .order_by(desc(Notification.created_at))\
-        .paginate(page=page, per_page=page_size, count=False, error_out=error_out)
+    results = db.session.query(Notification).filter(*filters).order_by(desc(Notification.created_at)).all()
     return results
 
 
+@statsd(namespace="dao")
 def dao_get_notification_by_reference(reference):
     return Notification.query.filter(
         Notification.reference == reference
     ).one()
 
 
+@statsd(namespace="dao")
 def dao_get_notification_history_by_reference(reference):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object
-        return Notification.query.filter(Notification.reference == reference).one()
+        return Notification.query.filter(
+            Notification.reference == reference
+        ).one()
     except NoResultFound:
-        return NotificationHistory.query.filter(NotificationHistory.reference == reference).one()
+        return NotificationHistory.query.filter(
+            NotificationHistory.reference == reference
+        ).one()
 
 
-def dao_get_notifications_processing_time_stats(start_date, end_date):
+@statsd(namespace="dao")
+def dao_get_notifications_by_references(references):
+    return Notification.query.filter(
+        Notification.reference.in_(references)
+    ).all()
+
+
+@statsd(namespace="dao")
+def dao_created_scheduled_notification(scheduled_notification):
+    db.session.add(scheduled_notification)
+    db.session.commit()
+
+
+@statsd(namespace="dao")
+def dao_get_scheduled_notifications():
+    notifications = Notification.query.join(
+        ScheduledNotification
+    ).filter(
+        ScheduledNotification.scheduled_for < datetime.utcnow(),
+        ScheduledNotification.pending).all()
+
+    return notifications
+
+
+def set_scheduled_notification_to_processed(notification_id):
+    db.session.query(ScheduledNotification).filter(
+        ScheduledNotification.notification_id == notification_id
+    ).update(
+        {'pending': False}
+    )
+    db.session.commit()
+
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
     """
-    For a given time range, returns the number of notifications sent and the number of
-    those notifications that we processed within 10 seconds
-
     SELECT
-    count(notifications),
+    count(notification_history),
     coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
     WHERE
     created_at > 'START DATE' AND
     created_at < 'END DATE' AND
     api_key_id IS NOT NULL AND
-    key_type != 'test';
+    key_type != 'test' AND
+    notification_type != 'letter';
     """
     under_10_secs = Notification.sent_at - Notification.created_at <= timedelta(seconds=10)
     sum_column = functions.coalesce(functions.sum(
@@ -523,9 +720,11 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
         Notification.created_at < end_date,
         Notification.api_key_id.isnot(None),
         Notification.key_type != KEY_TYPE_TEST,
+        Notification.notification_type != LETTER_TYPE
     ).one()
 
 
+@statsd(namespace="dao")
 def dao_get_last_notification_added_for_job_id(job_id):
     last_notification_added = Notification.query.filter(
         Notification.job_id == job_id
@@ -547,61 +746,60 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
+def dao_old_letters_with_created_status():
+    yesterday_bst = convert_utc_to_bst(datetime.utcnow()) - timedelta(days=1)
+    last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
+
+    notifications = Notification.query.filter(
+        Notification.created_at < convert_bst_to_utc(last_processing_deadline),
+        Notification.notification_type == LETTER_TYPE,
+        Notification.status == NOTIFICATION_CREATED
+    ).order_by(
+        Notification.created_at
+    ).all()
+    return notifications
+
+
+def letters_missing_from_sending_bucket(seconds_to_subtract):
+    older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
+    # We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero.
+    notifications = Notification.query.filter(
+        Notification.billable_units == 0,
+        Notification.updated_at == None,  # noqa
+        Notification.status == NOTIFICATION_CREATED,
+        Notification.created_at <= older_than_date,
+        Notification.notification_type == LETTER_TYPE,
+        Notification.key_type == KEY_TYPE_NORMAL
+    ).order_by(
+        Notification.created_at
+    ).all()
+
+    return notifications
+
+
+def dao_precompiled_letters_still_pending_virus_check():
+    ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
+
+    notifications = Notification.query.filter(
+        Notification.created_at < ninety_minutes_ago,
+        Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK
+    ).order_by(
+        Notification.created_at
+    ).all()
+    return notifications
+
+
 def _duplicate_update_warning(notification, status):
     current_app.logger.info(
         (
-            'Duplicate callback received for service {service_id}. '
-            'Notification ID {id} with type {type} sent by {sent_by}. '
-            'New status was {new_status}, current status is {old_status}. '
-            'This happened {time_diff} after being first set.'
+            'Duplicate callback received. Notification id {id} received a status update to {new_status}'
+            '{time_diff} after being set to {old_status}. {type} sent by {sent_by}'
         ).format(
             id=notification.id,
             old_status=notification.status,
             new_status=status,
             time_diff=datetime.utcnow() - (notification.updated_at or notification.created_at),
             type=notification.notification_type,
-            sent_by=notification.sent_by,
-            service_id=notification.service_id
+            sent_by=notification.sent_by
         )
     )
-
-
-def get_service_ids_with_notifications_before(notification_type, timestamp):
-    return {
-        row.service_id
-        for row in db.session.query(
-            Notification.service_id
-        ).filter(
-            Notification.notification_type == notification_type,
-            Notification.created_at < timestamp
-        ).distinct()
-    }
-
-
-def get_service_ids_with_notifications_on_date(notification_type, date):
-    start_date = get_local_midnight_in_utc(date)
-    end_date = get_local_midnight_in_utc(date + timedelta(days=1))
-
-    notification_table_query = db.session.query(
-        Notification.service_id.label('service_id')
-    ).filter(
-        Notification.notification_type == notification_type,
-        # using >= + < is much more efficient than date(created_at)
-        Notification.created_at >= start_date,
-        Notification.created_at < end_date,
-    )
-
-    # Looking at this table is more efficient for historical notifications,
-    # provided the task to populate it has run before they were archived.
-    ft_status_table_query = db.session.query(
-        FactNotificationStatus.service_id.label('service_id')
-    ).filter(
-        FactNotificationStatus.notification_type == notification_type,
-        FactNotificationStatus.local_date == date,
-    )
-
-    return {
-        row.service_id for row in db.session.query(union(
-            notification_table_query, ft_status_table_query
-        ).subquery()).distinct()
-    }

app/dao/organisation_dao.py

@@ -1,8 +1,14 @@
 from sqlalchemy.sql.expression import func
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.models import Domain, Organisation, Service, User
+from app.dao.dao_utils import VersionOptions, transactional, version_class
+from app.models import (
+    Organisation,
+    Domain,
+    InvitedOrganisationUser,
+    Service,
+    User
+)
 
 
 def dao_get_organisations():
@@ -11,7 +17,7 @@ def dao_get_organisations():
     ).all()
 
 
-def dao_count_organisations_with_live_services():
+def dao_count_organsations_with_live_services():
     return db.session.query(Organisation.id).join(Organisation.services).filter(
         Service.active.is_(True),
         Service.restricted.is_(False),
@@ -25,13 +31,6 @@ def dao_get_organisation_services(organisation_id):
     ).one().services
 
 
-def dao_get_organisation_live_services(organisation_id):
-    return Service.query.filter_by(
-        organisation_id=organisation_id,
-        restricted=False
-    ).all()
-
-
 def dao_get_organisation_by_id(organisation_id):
     return Organisation.query.filter_by(id=organisation_id).one()
 
@@ -55,12 +54,12 @@ def dao_get_organisation_by_service_id(service_id):
     return Organisation.query.join(Organisation.services).filter_by(id=service_id).first()
 
 
-@autocommit
+@transactional
 def dao_create_organisation(organisation):
     db.session.add(organisation)
 
 
-@autocommit
+@transactional
 def dao_update_organisation(organisation_id, **kwargs):
 
     domains = kwargs.pop('domains', None)
@@ -83,12 +82,12 @@ def dao_update_organisation(organisation_id, **kwargs):
     if 'organisation_type' in kwargs:
         _update_organisation_services(organisation, 'organisation_type', only_where_none=False)
 
-    if 'crown' in kwargs:
-        _update_organisation_services(organisation, 'crown', only_where_none=False)
-
     if 'email_branding_id' in kwargs:
         _update_organisation_services(organisation, 'email_branding')
 
+    if 'letter_branding_id' in kwargs:
+        _update_organisation_services(organisation, 'letter_branding')
+
     return num_updated
 
 
@@ -102,7 +101,7 @@ def _update_organisation_services(organisation, attribute, only_where_none=True)
         db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_add_service_to_organisation(service, organisation_id):
     organisation = Organisation.query.filter_by(
@@ -116,26 +115,21 @@ def dao_add_service_to_organisation(service, organisation_id):
     db.session.add(service)
 
 
+def dao_get_invited_organisation_user(user_id):
+    return InvitedOrganisationUser.query.filter_by(id=user_id).one()
+
+
 def dao_get_users_for_organisation(organisation_id):
-    return db.session.query(
-        User
-    ).join(
-        User.organisations
-    ).filter(
-        Organisation.id == organisation_id,
+    return User.query.filter(
+        User.organisations.any(id=organisation_id),
         User.state == 'active'
     ).order_by(User.created_at).all()
 
 
-@autocommit
+@transactional
 def dao_add_user_to_organisation(organisation_id, user_id):
     organisation = dao_get_organisation_by_id(organisation_id)
     user = User.query.filter_by(id=user_id).one()
     user.organisations.append(organisation)
     db.session.add(organisation)
     return user
-
-
-@autocommit
-def dao_remove_user_from_organisation(organisation, user):
-    organisation.users.remove(user)

app/dao/permissions_dao.py

@@ -1,15 +1,16 @@
 from app import db
 from app.dao import DAOClass
 from app.models import (
-    MANAGE_API_KEYS,
-    MANAGE_SETTINGS,
-    MANAGE_TEMPLATES,
-    MANAGE_USERS,
-    SEND_EMAILS,
-    SEND_TEXTS,
-    VIEW_ACTIVITY,
     Permission,
-)
+    MANAGE_USERS,
+    MANAGE_TEMPLATES,
+    MANAGE_SETTINGS,
+    SEND_TEXTS,
+    SEND_EMAILS,
+    SEND_LETTERS,
+    MANAGE_API_KEYS,
+    VIEW_ACTIVITY)
+
 
 # Default permissions for a service
 default_service_permissions = [
@@ -18,6 +19,7 @@ default_service_permissions = [
     MANAGE_SETTINGS,
     SEND_TEXTS,
     SEND_EMAILS,
+    SEND_LETTERS,
     MANAGE_API_KEYS,
     VIEW_ACTIVITY]
 

app/dao/provider_details_dao.py

@@ -1,18 +1,12 @@
 from datetime import datetime, timedelta
 
-from flask import current_app
-from notifications_utils.timezones import convert_utc_to_local_timezone
+from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import asc, desc, func
+from flask import current_app
 
+from app.dao.dao_utils import transactional
+from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, SMS_TYPE, User
 from app import db
-from app.dao.dao_utils import autocommit
-from app.models import (
-    SMS_TYPE,
-    FactBilling,
-    ProviderDetails,
-    ProviderDetailsHistory,
-    User,
-)
 
 
 def get_provider_details_by_id(provider_details_id):
@@ -24,8 +18,10 @@ def get_provider_details_by_identifier(identifier):
 
 
 def get_alternative_sms_provider(identifier):
-    if identifier == 'sns':
-        raise Exception("No alternative SMS providers currently available")
+    if identifier == 'firetext':
+        return 'mmg'
+    elif identifier == 'mmg':
+        return 'firetext'
     raise ValueError('Unrecognised sms provider {}'.format(identifier))
 
 
@@ -34,8 +30,6 @@ def dao_get_provider_versions(provider_id):
         id=provider_id
     ).order_by(
         desc(ProviderDetailsHistory.version)
-    ).limit(
-        100  # limit results instead of adding pagination
     ).all()
 
 
@@ -75,7 +69,7 @@ def _get_sms_providers_for_update(time_threshold):
     return q
 
 
-@autocommit
+@transactional
 def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     """
     Will reduce a chosen sms provider's priority, and increase the other provider's priority by 10 points each.
@@ -84,8 +78,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     amount_to_reduce_by = 10
     providers_list = _get_sms_providers_for_update(time_threshold)
 
-    if len(providers_list) < 2:
-        current_app.logger.info("Not adjusting providers, number of active providers is less than 2.")
+    if not providers_list:
         return
 
     providers = {provider.identifier: provider for provider in providers_list}
@@ -102,7 +95,7 @@ def dao_reduce_sms_provider_priority(identifier, *, time_threshold):
     _adjust_provider_priority(increased_provider, increased_provider_priority)
 
 
-@autocommit
+@transactional
 def dao_adjust_provider_priority_back_to_resting_points():
     """
     Provided that neither SMS provider has been modified in the last hour, move both providers by 10 percentage points
@@ -136,7 +129,7 @@ def get_provider_details_by_notification_type(notification_type, supports_intern
     return ProviderDetails.query.filter(*filters).order_by(asc(ProviderDetails.priority)).all()
 
 
-@autocommit
+@transactional
 def dao_update_provider_details(provider_details):
     _update_provider_details_without_commit(provider_details)
 
@@ -155,15 +148,15 @@ def _update_provider_details_without_commit(provider_details):
 def dao_get_provider_stats():
     # this query does not include the current day since the task to populate ft_billing runs overnight
 
-    current_local_datetime = convert_utc_to_local_timezone(datetime.utcnow())
-    first_day_of_the_month = current_local_datetime.date().replace(day=1)
+    current_bst_datetime = convert_utc_to_bst(datetime.utcnow())
+    first_day_of_the_month = current_bst_datetime.date().replace(day=1)
 
     subquery = db.session.query(
         FactBilling.provider,
         func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('current_month_billable_sms')
     ).filter(
         FactBilling.notification_type == SMS_TYPE,
-        FactBilling.local_date >= first_day_of_the_month
+        FactBilling.bst_date >= first_day_of_the_month
     ).group_by(
         FactBilling.provider
     ).subquery()

app/dao/provider_rates_dao.py

@@ -0,0 +1,11 @@
+from app.models import ProviderRates, ProviderDetails
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
+    db.session.add(provider_rates)

app/dao/returned_letters_dao.py

@@ -0,0 +1,98 @@
+from datetime import datetime
+
+from sqlalchemy import func, desc
+from sqlalchemy.dialects.postgresql import insert
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import (
+    Job,
+    Notification,
+    NotificationHistory,
+    ReturnedLetter,
+    Template,
+    User,
+)
+
+
+def _get_notification_ids_for_references(references):
+    notification_ids = db.session.query(Notification.id, Notification.service_id).filter(
+        Notification.reference.in_(references)
+    ).all()
+
+    notification_history_ids = db.session.query(NotificationHistory.id, NotificationHistory.service_id).filter(
+        NotificationHistory.reference.in_(references)
+    ).all()
+
+    return notification_ids + notification_history_ids
+
+
+@transactional
+def insert_or_update_returned_letters(references):
+    data = _get_notification_ids_for_references(references)
+    for row in data:
+        table = ReturnedLetter.__table__
+
+        stmt = insert(table).values(
+            reported_at=datetime.utcnow().date(),
+            service_id=row.service_id,
+            notification_id=row.id,
+            created_at=datetime.utcnow()
+        )
+
+        stmt = stmt.on_conflict_do_update(
+            index_elements=[table.c.notification_id],
+            set_={
+                'reported_at': datetime.utcnow().date(),
+                'updated_at': datetime.utcnow()
+            }
+        )
+        db.session.connection().execute(stmt)
+
+
+def fetch_returned_letter_summary(service_id):
+    return db.session.query(
+        func.count(ReturnedLetter.notification_id).label('returned_letter_count'),
+        ReturnedLetter.reported_at
+    ).filter(
+        ReturnedLetter.service_id == service_id,
+    ).group_by(
+        ReturnedLetter.reported_at
+    ).order_by(
+        desc(ReturnedLetter.reported_at)
+    ).all()
+
+
+def fetch_returned_letters(service_id, report_date):
+    results = []
+    for table in [Notification, NotificationHistory]:
+        query = db.session.query(
+            ReturnedLetter.notification_id,
+            ReturnedLetter.reported_at,
+            table.client_reference,
+            table.created_at,
+            Template.name.label('template_name'),
+            table.template_id,
+            table.template_version,
+            Template.hidden,
+            table.api_key_id,
+            table.created_by_id,
+            User.name.label('user_name'),
+            User.email_address,
+            Job.original_file_name,
+            (table.job_row_number + 1).label('job_row_number')  # row numbers start at 0
+        ).outerjoin(
+            User, table.created_by_id == User.id
+        ).outerjoin(
+            Job, table.job_id == Job.id
+        ).filter(
+            ReturnedLetter.service_id == service_id,
+            ReturnedLetter.reported_at == report_date,
+            ReturnedLetter.notification_id == table.id,
+            table.template_id == Template.id
+        ).order_by(
+            desc(ReturnedLetter.reported_at), desc(table.created_at)
+        )
+        results = results + query.all()
+    results = sorted(results, key=lambda i: i.created_at, reverse=True)
+    return results

app/dao/service_callback_api_dao.py

@@ -1,15 +1,13 @@
 from datetime import datetime
 
-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
-from app.models import (
-    COMPLAINT_CALLBACK_TYPE,
-    DELIVERY_STATUS_CALLBACK_TYPE,
-    ServiceCallbackApi,
-)
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
+from app.models import ServiceCallbackApi
+
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def save_service_callback_api(service_callback_api):
     service_callback_api.id = create_uuid()
@@ -17,7 +15,7 @@ def save_service_callback_api(service_callback_api):
     db.session.add(service_callback_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceCallbackApi)
 def reset_service_callback_api(service_callback_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -48,6 +46,6 @@ def get_service_complaint_callback_api_for_service(service_id):
     ).first()
 
 
-@autocommit
+@transactional
 def delete_service_callback_api(service_callback_api):
     db.session.delete(service_callback_api)

app/dao/service_contact_list_dao.py

@@ -1,33 +0,0 @@
-from app import db
-from app.models import ServiceContactList
-
-
-def dao_get_contact_list_by_id(service_id, contact_list_id):
-    contact_list = ServiceContactList.query.filter_by(
-        service_id=service_id,
-        id=contact_list_id,
-        archived=False,
-    ).one()
-
-    return contact_list
-
-
-def dao_get_contact_lists(service_id):
-    contact_lists = ServiceContactList.query.filter_by(
-        service_id=service_id,
-        archived=False,
-    ).order_by(
-        ServiceContactList.created_at.desc()
-    )
-    return contact_lists.all()
-
-
-def save_service_contact_list(service_contact_list):
-    db.session.add(service_contact_list)
-    db.session.commit()
-
-
-def dao_archive_contact_list(service_contact_list):
-    service_contact_list.archived = True
-    db.session.add(service_contact_list)
-    db.session.commit()

app/dao/service_data_retention_dao.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceDataRetention
 
 
@@ -28,7 +28,7 @@ def fetch_service_data_retention_by_notification_type(service_id, notification_t
     return data_retention_list
 
 
-@autocommit
+@transactional
 def insert_service_data_retention(service_id, notification_type, days_of_retention):
     new_data_retention = ServiceDataRetention(service_id=service_id,
                                               notification_type=notification_type,
@@ -38,7 +38,7 @@ def insert_service_data_retention(service_id, notification_type, days_of_retenti
     return new_data_retention
 
 
-@autocommit
+@transactional
 def update_service_data_retention(service_data_retention_id, service_id, days_of_retention):
     updated_count = ServiceDataRetention.query.filter(
         ServiceDataRetention.id == service_data_retention_id,
@@ -50,7 +50,3 @@ def update_service_data_retention(service_data_retention_id, service_id, days_of
         }
     )
     return updated_count
-
-
-def fetch_service_data_retention_for_all_services_by_notification_type(notification_type):
-    return ServiceDataRetention.query.filter(ServiceDataRetention.notification_type == notification_type).all()

app/dao/service_email_reply_to_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceEmailReplyTo
@@ -28,7 +28,7 @@ def dao_get_reply_to_by_id(service_id, reply_to_id):
     return reply_to
 
 
-@autocommit
+@transactional
 def add_reply_to_email_address_for_service(service_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -41,7 +41,7 @@ def add_reply_to_email_address_for_service(service_id, email_address, is_default
     return new_reply_to
 
 
-@autocommit
+@transactional
 def update_reply_to_email_address(service_id, reply_to_id, email_address, is_default):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -57,7 +57,7 @@ def update_reply_to_email_address(service_id, reply_to_id, email_address, is_def
     return reply_to_update
 
 
-@autocommit
+@transactional
 def archive_reply_to_email_address(service_id, reply_to_id):
     reply_to_archive = ServiceEmailReplyTo.query.filter_by(
         id=reply_to_id,

app/dao/service_guest_list_dao.py

@@ -1,17 +0,0 @@
-from app import db
-from app.models import ServiceGuestList
-
-
-def dao_fetch_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).all()
-
-
-def dao_add_and_commit_guest_list_contacts(objs):
-    db.session.add_all(objs)
-    db.session.commit()
-
-
-def dao_remove_service_guest_list(service_id):
-    return ServiceGuestList.query.filter(
-        ServiceGuestList.service_id == service_id).delete()

app/dao/service_inbound_api_dao.py

@@ -1,11 +1,11 @@
 from datetime import datetime
 
-from app import create_uuid, db
-from app.dao.dao_utils import autocommit, version_class
+from app import db, create_uuid
+from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceInboundApi
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def save_service_inbound_api(service_inbound_api):
     service_inbound_api.id = create_uuid()
@@ -13,7 +13,7 @@ def save_service_inbound_api(service_inbound_api):
     db.session.add(service_inbound_api)
 
 
-@autocommit
+@transactional
 @version_class(ServiceInboundApi)
 def reset_service_inbound_api(service_inbound_api, updated_by_id, url=None, bearer_token=None):
     if url:
@@ -35,6 +35,6 @@ def get_service_inbound_api_for_service(service_id):
     return ServiceInboundApi.query.filter_by(service_id=service_id).first()
 
 
-@autocommit
+@transactional
 def delete_service_inbound_api(service_inbound_api):
     db.session.delete(service_inbound_api)

app/dao/service_letter_contact_dao.py

@@ -0,0 +1,105 @@
+from sqlalchemy import desc
+
+from app import db
+from app.dao.dao_utils import transactional
+from app.models import ServiceLetterContact, Template
+
+
+def dao_get_letter_contacts_by_service_id(service_id):
+    letter_contacts = db.session.query(
+        ServiceLetterContact
+    ).filter(
+        ServiceLetterContact.service_id == service_id,
+        ServiceLetterContact.archived == False  # noqa
+    ).order_by(
+        desc(ServiceLetterContact.is_default),
+        desc(ServiceLetterContact.created_at)
+    ).all()
+
+    return letter_contacts
+
+
+def dao_get_letter_contact_by_id(service_id, letter_contact_id):
+    letter_contact = db.session.query(
+        ServiceLetterContact
+    ).filter(
+        ServiceLetterContact.service_id == service_id,
+        ServiceLetterContact.id == letter_contact_id,
+        ServiceLetterContact.archived == False  # noqa
+    ).one()
+    return letter_contact
+
+
+@transactional
+def add_letter_contact_for_service(service_id, contact_block, is_default):
+    old_default = _get_existing_default(service_id)
+    if is_default:
+        _reset_old_default_to_false(old_default)
+
+    new_letter_contact = ServiceLetterContact(
+        service_id=service_id,
+        contact_block=contact_block,
+        is_default=is_default
+    )
+    db.session.add(new_letter_contact)
+    return new_letter_contact
+
+
+@transactional
+def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
+    old_default = _get_existing_default(service_id)
+    # if we want to make this the default, ensure there are no other existing defaults
+    if is_default:
+        _reset_old_default_to_false(old_default)
+
+    letter_contact_update = ServiceLetterContact.query.get(letter_contact_id)
+    letter_contact_update.contact_block = contact_block
+    letter_contact_update.is_default = is_default
+    db.session.add(letter_contact_update)
+    return letter_contact_update
+
+
+@transactional
+def archive_letter_contact(service_id, letter_contact_id):
+    letter_contact_to_archive = ServiceLetterContact.query.filter_by(
+        id=letter_contact_id,
+        service_id=service_id
+    ).one()
+
+    Template.query.filter_by(
+        service_letter_contact_id=letter_contact_id
+    ).update({
+        'service_letter_contact_id': None
+    })
+
+    letter_contact_to_archive.archived = True
+
+    db.session.add(letter_contact_to_archive)
+    return letter_contact_to_archive
+
+
+def _get_existing_default(service_id):
+    old_defaults = [
+        x for x
+        in dao_get_letter_contacts_by_service_id(service_id=service_id)
+        if x.is_default
+    ]
+
+    if len(old_defaults) == 0:
+        return None
+
+    if len(old_defaults) == 1:
+        return old_defaults[0]
+
+    raise Exception(
+        "There should only be one default letter contact for each service. Service {} has {}".format(
+            service_id,
+            len(old_defaults)
+        )
+    )
+
+
+def _reset_old_default_to_false(old_default):
+    if old_default:
+        old_default.is_default = False
+        db.session.add(old_default)

app/dao/service_permissions_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServicePermission
 
 
@@ -8,7 +8,7 @@ def dao_fetch_service_permissions(service_id):
         ServicePermission.service_id == service_id).all()
 
 
-@autocommit
+@transactional
 def dao_add_service_permission(service_id, permission):
     service_permission = ServicePermission(service_id=service_id, permission=permission)
     db.session.add(service_permission)

app/dao/service_sms_sender_dao.py

@@ -1,7 +1,7 @@
 from sqlalchemy import desc
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.exceptions import ArchiveValidationError
 from app.models import ServiceSmsSender
 
@@ -32,7 +32,7 @@ def dao_get_sms_senders_by_service_id(service_id):
     ).order_by(desc(ServiceSmsSender.is_default)).all()
 
 
-@autocommit
+@transactional
 def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_number_id=None):
     old_default = _get_existing_default(service_id=service_id)
     if is_default:
@@ -51,7 +51,7 @@ def dao_add_sms_sender_for_service(service_id, sms_sender, is_default, inbound_n
     return new_sms_sender
 
 
-@autocommit
+@transactional
 def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default, sms_sender=None):
     old_default = _get_existing_default(service_id)
     if is_default:
@@ -68,7 +68,7 @@ def dao_update_service_sms_sender(service_id, service_sms_sender_id, is_default,
     return sms_sender_to_update
 
 
-@autocommit
+@transactional
 def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sender, inbound_number_id):
     service_sms_sender.sms_sender = sms_sender
     service_sms_sender.inbound_number_id = inbound_number_id
@@ -76,7 +76,7 @@ def update_existing_sms_sender_with_inbound_number(service_sms_sender, sms_sende
     return service_sms_sender
 
 
-@autocommit
+@transactional
 def archive_sms_sender(service_id, sms_sender_id):
     sms_sender_to_archive = ServiceSmsSender.query.filter_by(
         id=sms_sender_id,

app/dao/service_user_dao.py

@@ -1,6 +1,6 @@
 
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import ServiceUser, User
 
 
@@ -9,13 +9,9 @@ def dao_get_service_user(user_id, service_id):
 
 
 def dao_get_active_service_users(service_id):
-    query = db.session.query(
-        ServiceUser
-    ).join(
-        User, User.id == ServiceUser.user_id
-    ).filter(
-        User.state == 'active',
-        ServiceUser.service_id == service_id
+    query = ServiceUser.query.join(ServiceUser.user).filter(
+        ServiceUser.service_id == service_id,
+        User.state == 'active'
     )
 
     return query.all()
@@ -25,6 +21,6 @@ def dao_get_service_users_by_user_id(user_id):
     return ServiceUser.query.filter_by(user_id=user_id).all()
 
 
-@autocommit
+@transactional
 def dao_update_service_user(service_user):
     db.session.add(service_user)

app/dao/service_whitelist_dao.py

@@ -0,0 +1,17 @@
+from app import db
+from app.models import ServiceWhitelist
+
+
+def dao_fetch_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).all()
+
+
+def dao_add_and_commit_whitelisted_contacts(objs):
+    db.session.add_all(objs)
+    db.session.commit()
+
+
+def dao_remove_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).delete()

app/dao/services_dao.py

@@ -1,24 +1,26 @@
 import uuid
 from datetime import date, datetime, timedelta
 
-from flask import current_app
-from sqlalchemy import Float, cast
+from notifications_utils.statsd_decorators import statsd
+from sqlalchemy.sql.expression import asc, case, and_, func
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import and_, asc, case, func
+from sqlalchemy import cast, Float
+from flask import current_app
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.email_branding_dao import dao_get_email_branding_by_name
+from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    EMAIL_TYPE,
-    INTERNATIONAL_SMS_TYPE,
-    KEY_TYPE_TEST,
-    NOTIFICATION_PERMANENT_FAILURE,
-    SMS_TYPE,
     AnnualBilling,
     ApiKey,
     FactBilling,
@@ -30,8 +32,6 @@ from app.models import (
     Organisation,
     Permission,
     Service,
-    ServiceContactList,
-    ServiceEmailReplyTo,
     ServicePermission,
     ServiceSmsSender,
     Template,
@@ -39,16 +39,22 @@ from app.models import (
     TemplateRedacted,
     User,
     VerifyCode,
+    CROWN_ORGANISATION_TYPES,
+    EMAIL_TYPE,
+    INTERNATIONAL_SMS_TYPE,
+    KEY_TYPE_TEST,
+    NHS_ORGANISATION_TYPES,
+    NON_CROWN_ORGANISATION_TYPES,
+    NOTIFICATION_PERMANENT_FAILURE,
+    SMS_TYPE,
+    LETTER_TYPE,
 )
-from app.utils import (
-    escape_special_characters,
-    get_archived_db_column_value,
-    get_local_midnight_in_utc,
-)
+from app.utils import email_address_is_nhs, escape_special_characters, get_london_midnight_in_utc, midnight_n_days_ago
 
 DEFAULT_SERVICE_PERMISSIONS = [
     SMS_TYPE,
     EMAIL_TYPE,
+    LETTER_TYPE,
     INTERNATIONAL_SMS_TYPE,
 ]
 
@@ -90,8 +96,8 @@ def dao_fetch_live_services_data():
     ).subquery()
 
     this_year_ft_billing = FactBilling.query.filter(
-        FactBilling.local_date >= year_start_date,
-        FactBilling.local_date <= year_end_date,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
     ).subquery()
 
     data = db.session.query(
@@ -106,12 +112,16 @@ def dao_fetch_live_services_data():
         Service.go_live_at.label("live_date"),
         Service.volume_sms.label('sms_volume_intent'),
         Service.volume_email.label('email_volume_intent'),
+        Service.volume_letter.label('letter_volume_intent'),
         case([
             (this_year_ft_billing.c.notification_type == 'email', func.sum(this_year_ft_billing.c.notifications_sent))
         ], else_=0).label("email_totals"),
         case([
             (this_year_ft_billing.c.notification_type == 'sms', func.sum(this_year_ft_billing.c.notifications_sent))
         ], else_=0).label("sms_totals"),
+        case([
+            (this_year_ft_billing.c.notification_type == 'letter', func.sum(this_year_ft_billing.c.notifications_sent))
+        ], else_=0).label("letter_totals"),
         AnnualBilling.free_sms_fragment_limit,
     ).join(
         Service.annual_billing
@@ -145,6 +155,7 @@ def dao_fetch_live_services_data():
         Service.go_live_at,
         Service.volume_sms,
         Service.volume_email,
+        Service.volume_letter,
         this_year_ft_billing.c.notification_type,
         AnnualBilling.free_sms_fragment_limit,
     ).order_by(
@@ -157,6 +168,7 @@ def dao_fetch_live_services_data():
         if existing_service is not None:
             existing_service["email_totals"] += row.email_totals
             existing_service["sms_totals"] += row.sms_totals
+            existing_service["letter_totals"] += row.letter_totals
         else:
             results.append(row._asdict())
     return results
@@ -217,17 +229,7 @@ def dao_fetch_all_services_by_user(user_id, only_active=False):
     return query.all()
 
 
-def dao_fetch_all_services_created_by_user(user_id):
-    query = Service.query.filter_by(
-        created_by_id=user_id
-    ).order_by(
-        asc(Service.created_at)
-    )
-
-    return query.all()
-
-
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -243,8 +245,8 @@ def dao_archive_service(service_id):
     ).filter(Service.id == service_id).one()
 
     service.active = False
-    service.name = get_archived_db_column_value(service.name)
-    service.email_from = get_archived_db_column_value(service.email_from)
+    service.name = '_archived_' + service.name
+    service.email_from = '_archived_' + service.email_from
 
     for template in service.templates:
         if not template.archived:
@@ -264,7 +266,7 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
     ).one()
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_create_service(
     service,
@@ -272,6 +274,9 @@ def dao_create_service(
     service_id=None,
     service_permissions=None,
 ):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.
 
     if not user:
         raise ValueError("Can't create a service without a user")
@@ -298,18 +303,27 @@ def dao_create_service(
     if organisation:
         service.organisation_id = organisation.id
         service.organisation_type = organisation.organisation_type
-
         if organisation.email_branding:
             service.email_branding = organisation.email_branding
 
+        if organisation.letter_branding and not service.letter_branding:
+            service.letter_branding = organisation.letter_branding
+
+    elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
+        service.email_branding = dao_get_email_branding_by_name('NHS')
+        service.letter_branding = dao_get_letter_branding_by_name('NHS')
     if organisation:
         service.crown = organisation.crown
+    elif service.organisation_type in CROWN_ORGANISATION_TYPES:
+        service.crown = True
+    elif service.organisation_type in NON_CROWN_ORGANISATION_TYPES:
+        service.crown = False
     service.count_as_live = not user.platform_admin
 
     db.session.add(service)
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_update_service(service):
     db.session.add(service)
@@ -361,8 +375,6 @@ def delete_service_and_all_associated_db_objects(service):
     _delete_commit(TemplateRedacted.query.filter(TemplateRedacted.template_id.in_(subq)))
 
     _delete_commit(ServiceSmsSender.query.filter_by(service=service))
-    _delete_commit(ServiceEmailReplyTo.query.filter_by(service=service))
-    _delete_commit(ServiceContactList.query.filter_by(service=service))
     _delete_commit(InvitedUser.query.filter_by(service=service))
     _delete_commit(Permission.query.filter_by(service=service))
     _delete_commit(NotificationHistory.query.filter_by(service=service))
@@ -379,39 +391,64 @@ def delete_service_and_all_associated_db_objects(service):
     list(map(db.session.delete, verify_codes))
     db.session.commit()
     users = [x for x in service.users]
-    for user in users:
-        user.organisations = []
-        service.users.remove(user)
+    map(service.users.remove, users)
+    [service.users.remove(x) for x in users]
     _delete_commit(Service.get_history_model().query.filter_by(id=service.id))
     db.session.delete(service)
     db.session.commit()
-    for user in users:
-        db.session.delete(user)
+    list(map(db.session.delete, users))
     db.session.commit()
 
 
-def dao_fetch_todays_stats_for_service(service_id):
-    today = date.today()
-    start_date = get_local_midnight_in_utc(today)
+@statsd(namespace="dao")
+def dao_fetch_stats_for_service(service_id, limit_days):
+    # We always want between seven and eight days
+    start_date = midnight_n_days_ago(limit_days)
+    return _stats_for_service_query(service_id).filter(
+        Notification.created_at >= start_date
+    ).all()
 
+
+@statsd(namespace="dao")
+def dao_fetch_todays_stats_for_service(service_id):
+    return _stats_for_service_query(service_id).filter(
+        func.date(Notification.created_at) == date.today()
+    ).all()
+
+
+def fetch_todays_total_message_count(service_id):
+    result = db.session.query(
+        func.count(Notification.id).label('count')
+    ).filter(
+        Notification.service_id == service_id,
+        Notification.key_type != KEY_TYPE_TEST,
+        func.date(Notification.created_at) == date.today()
+    ).group_by(
+        Notification.notification_type,
+        Notification.status,
+    ).first()
+    return 0 if result is None else result.count
+
+
+def _stats_for_service_query(service_id):
     return db.session.query(
         Notification.notification_type,
         Notification.status,
         func.count(Notification.id).label('count')
     ).filter(
         Notification.service_id == service_id,
-        Notification.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= start_date
+        Notification.key_type != KEY_TYPE_TEST
     ).group_by(
         Notification.notification_type,
         Notification.status,
-    ).all()
+    )
 
 
+@statsd(namespace='dao')
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True):
     today = date.today()
-    start_date = get_local_midnight_in_utc(today)
-    end_date = get_local_midnight_in_utc(today + timedelta(days=1))
+    start_date = get_london_midnight_in_utc(today)
+    end_date = get_london_midnight_in_utc(today + timedelta(days=1))
 
     subquery = db.session.query(
         Notification.notification_type,
@@ -453,7 +490,7 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act
     return query.all()
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(ApiKey, must_write_history=False),
     VersionOptions(Service),
@@ -472,7 +509,7 @@ def dao_suspend_service(service_id):
     service.active = False
 
 
-@autocommit
+@transactional
 @version_class(Service)
 def dao_resume_service(service_id):
     service = Service.query.get(service_id)
@@ -500,8 +537,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
         Notification.notification_type == SMS_TYPE,
         func.substr(Notification.normalised_to, 3, 7) == '7700900',
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -509,7 +546,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
     ).all()
 
 
-def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000):
+def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=100):
     subquery = db.session.query(
         func.count(Notification.id).label('total_count'),
         Notification.service_id.label('service_id')
@@ -520,8 +557,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.key_type != KEY_TYPE_TEST,
         Notification.notification_type == SMS_TYPE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
     ).having(
@@ -546,8 +583,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
         Notification.notification_type == SMS_TYPE,
         Notification.status == NOTIFICATION_PERMANENT_FAILURE,
         Service.restricted == False,  # noqa
-        Service.research_mode == False,  # noqa
-        Service.active == True,  # noqa
+        Service.research_mode == False,
+        Service.active == True,
     ).group_by(
         Notification.service_id,
         subquery.c.total_count
@@ -556,23 +593,3 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
     )
 
     return query.all()
-
-
-def get_live_services_with_organisation():
-    query = db.session.query(
-        Service.id.label("service_id"),
-        Service.name.label("service_name"),
-        Organisation.id.label("organisation_id"),
-        Organisation.name.label("organisation_name")
-    ).outerjoin(
-        Service.organisation
-    ).filter(
-        Service.count_as_live.is_(True),
-        Service.active.is_(True),
-        Service.restricted.is_(False)
-    ).order_by(
-        Organisation.name,
-        Service.name
-    )
-
-    return query.all()

app/dao/template_folder_dao.py

@@ -1,5 +1,5 @@
 from app import db
-from app.dao.dao_utils import autocommit
+from app.dao.dao_utils import transactional
 from app.models import TemplateFolder
 
 
@@ -14,16 +14,16 @@ def dao_get_valid_template_folders_by_id(folder_ids):
     return TemplateFolder.query.filter(TemplateFolder.id.in_(folder_ids)).all()
 
 
-@autocommit
+@transactional
 def dao_create_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_update_template_folder(template_folder):
     db.session.add(template_folder)
 
 
-@autocommit
+@transactional
 def dao_delete_template_folder(template_folder):
     db.session.delete(template_folder)

app/dao/templates_dao.py

@@ -1,14 +1,26 @@
-import uuid
 from datetime import datetime
+import uuid
 
+from flask import current_app
 from sqlalchemy import asc, desc
 
 from app import db
-from app.dao.dao_utils import VersionOptions, autocommit, version_class
-from app.models import Template, TemplateHistory, TemplateRedacted
+from app.models import (
+    LETTER_TYPE,
+    SECOND_CLASS,
+    Template,
+    TemplateHistory,
+    TemplateRedacted
+)
+from app.dao.dao_utils import (
+    transactional,
+    version_class,
+    VersionOptions,
+)
+from app.dao.users_dao import get_user_by_id
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
@@ -30,15 +42,49 @@ def dao_create_template(template):
     db.session.add(template)
 
 
-@autocommit
+@transactional
 @version_class(
     VersionOptions(Template, history_class=TemplateHistory)
 )
 def dao_update_template(template):
+    if template.archived:
+        template.folder = None
+
     db.session.add(template)
 
 
-@autocommit
+@transactional
+def dao_update_template_reply_to(template_id, reply_to):
+    Template.query.filter_by(id=template_id).update(
+        {"service_letter_contact_id": reply_to,
+         "updated_at": datetime.utcnow(),
+         "version": Template.version + 1,
+         }
+    )
+    template = Template.query.filter_by(id=template_id).one()
+
+    history = TemplateHistory(**
+                              {
+                                  "id": template.id,
+                                  "name": template.name,
+                                  "template_type": template.template_type,
+                                  "created_at": template.created_at,
+                                  "updated_at": template.updated_at,
+                                  "content": template.content,
+                                  "service_id": template.service_id,
+                                  "subject": template.subject,
+                                  "postage": template.postage,
+                                  "created_by_id": template.created_by_id,
+                                  "version": template.version,
+                                  "archived": template.archived,
+                                  "process_type": template.process_type,
+                                  "service_letter_contact_id": template.service_letter_contact_id
+                              })
+    db.session.add(history)
+    return template
+
+
+@transactional
 def dao_redact_template(template, user_id):
     template.template_redacted.redact_personalisation = True
     template.template_redacted.updated_at = datetime.utcnow()
@@ -93,3 +139,28 @@ def dao_get_template_versions(service_id, template_id):
     ).order_by(
         desc(TemplateHistory.version)
     ).all()
+
+
+def get_precompiled_letter_template(service_id):
+    template = Template.query.filter_by(
+        service_id=service_id,
+        template_type=LETTER_TYPE,
+        hidden=True
+    ).first()
+    if template is not None:
+        return template
+
+    template = Template(
+        name='Pre-compiled PDF',
+        created_by=get_user_by_id(current_app.config['NOTIFY_USER_ID']),
+        service_id=service_id,
+        template_type=LETTER_TYPE,
+        hidden=True,
+        subject='Pre-compiled PDF',
+        content='',
+        postage=SECOND_CLASS
+    )
+
+    dao_create_template(template)
+
+    return template

app/dao/uploads_dao.py

@@ -1,58 +1,22 @@
-from datetime import datetime
-from os import getenv
-
 from flask import current_app
-from sqlalchemy import String, and_, desc, func, literal, text
+from sqlalchemy import desc, literal
 
 from app import db
 from app.models import (
-    JOB_STATUS_CANCELLED,
-    JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
-    NOTIFICATION_CANCELLED,
-    Job,
-    Notification,
-    ServiceDataRetention,
-    Template,
+    Job, Notification, Template, LETTER_TYPE, JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED,
+    NOTIFICATION_CANCELLED
 )
 from app.utils import midnight_n_days_ago
 
 
-def _get_printing_day(created_at):
-    return func.date_trunc(
-        'day',
-        func.timezone(getenv("TIMEZONE", "America/New_York"), func.timezone('UTC', created_at)) + text(
-            # We add 6 hours 30 minutes to the local created_at time so that
-            # any letters created after 5:30pm get shifted into the next day
-            "interval '6 hours 30 minutes'"
-        )
-    )
-
-
-def _get_printing_datetime(created_at):
-    return _get_printing_day(created_at) + text(
-        # Letters are printed from 5:30pm each day
-        "interval '17 hours 30 minutes'"
-    )
-
-
-def _naive_gmt_to_utc(column):
-    return func.timezone('UTC', func.timezone(getenv("TIMEZONE", "America/New_York"), column))
-
-
 def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size=50):
     # Hardcoded filter to exclude cancelled or scheduled jobs
     # for the moment, but we may want to change this method take 'statuses' as a argument in the future
-    today = datetime.utcnow().date()
     jobs_query_filter = [
         Job.service_id == service_id,
         Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
         Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
-        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
-        func.coalesce(
-            Job.processing_started, Job.created_at
-        ) >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
-        Job.contact_list_id.is_(None),
+        Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED])
     ]
     if limit_days is not None:
         jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days))
@@ -61,21 +25,11 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Job.id,
         Job.original_file_name,
         Job.notification_count,
-        Template.template_type,
-        func.coalesce(ServiceDataRetention.days_of_retention, 7).label('days_of_retention'),
         Job.created_at.label("created_at"),
         Job.scheduled_for.label("scheduled_for"),
         Job.processing_started.label('processing_started'),
         Job.job_status.label("status"),
-        literal('job').label('upload_type'),
-        literal(None).label('recipient'),
-    ).join(
-        Template, Job.template_id == Template.id
-    ).outerjoin(
-        ServiceDataRetention, and_(
-            Template.service_id == ServiceDataRetention.service_id,
-            func.cast(Template.template_type, String) == func.cast(ServiceDataRetention.notification_type, String)
-        )
+        literal('job').label('upload_type')
     ).filter(
         *jobs_query_filter
     )
@@ -85,43 +39,26 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
         Notification.notification_type == LETTER_TYPE,
         Notification.api_key_id == None,  # noqa
         Notification.status != NOTIFICATION_CANCELLED,
-        Template.hidden == True,  # noqa
-        Notification.created_at >= today - func.coalesce(ServiceDataRetention.days_of_retention, 7)
+        Template.hidden == True,
+
     ]
     if limit_days is not None:
         letters_query_filter.append(Notification.created_at >= midnight_n_days_ago(limit_days))
 
-    letters_subquery = db.session.query(
-        func.count().label('notification_count'),
-        _naive_gmt_to_utc(_get_printing_datetime(Notification.created_at)).label('printing_at'),
+    letters_query = db.session.query(
+        Notification.id,
+        Notification.client_reference.label('original_file_name'),
+        literal('1').label('notification_count'),
+        Notification.created_at.label("created_at"),
+        literal(None).label('scheduled_for'),
+        # letters don't have a processing_started date but we want created_at to be used for sorting
+        Notification.created_at.label('processing_started'),
+        Notification.status,
+        literal('letter').label('upload_type')
     ).join(
         Template, Notification.template_id == Template.id
-    ).outerjoin(
-        ServiceDataRetention, and_(
-            Template.service_id == ServiceDataRetention.service_id,
-            func.cast(Template.template_type, String) == func.cast(ServiceDataRetention.notification_type, String)
-        )
     ).filter(
         *letters_query_filter
-    ).group_by(
-        'printing_at'
-    ).subquery()
-
-    letters_query = db.session.query(
-        literal(None).label('id'),
-        literal('Uploaded letters').label('original_file_name'),
-        letters_subquery.c.notification_count.label('notification_count'),
-        literal('letter').label('template_type'),
-        literal(None).label('days_of_retention'),
-        letters_subquery.c.printing_at.label('created_at'),
-        literal(None).label('scheduled_for'),
-        letters_subquery.c.printing_at.label('processing_started'),
-        literal(None).label('status'),
-        literal('letter_day').label('upload_type'),
-        literal(None).label('recipient'),
-    ).group_by(
-        letters_subquery.c.notification_count,
-        letters_subquery.c.printing_at,
     )
 
     return jobs_query.union_all(